{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost Parameter Tuning for Rental Listing Inquiries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1、数据读取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train = pd.read_csv(\"RentListingInquries_FE_train.csv\")\n",
    "data_test = pd.read_csv(\"RentListingInquries_FE_test.csv\")\n",
    "#data_train.info()\n",
    "#data_train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = data_train['interest_level']\n",
    "X_train = data_train.drop([\"interest_level\"], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.series.Series"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "test数据无标签"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2、模型训练：超参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "#5折交叉验证，样本均衡随机排序\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1、初步确定弱学习器数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#直接调用xgboost内嵌的交叉验证（cv），可对连续的n_estimators参数进行快速交叉验证，比GridSearchCV快很多\n",
    "#而GridSearchCV只能对有限个参数进行交叉验证\n",
    "#输入xgb实例、样本特征、标签、kfold对象、提前终止条件\n",
    "#对实例的弱学习器数目进行调参，保存结果，利用数据对调参后的模型进行训练\n",
    "def modelfit(alg, X_train, y_train, cv_folds=None, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 3\n",
    "    #3类分\n",
    "      \n",
    "    #直接调用xgboost，而非sklarn的wrapper类\n",
    "    xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "      \n",
    "    cvresult.to_csv('nestimators_first.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators = cvresult.shape[0]    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    #train_predprob = alg.predict_proba(X_train)\n",
    "    #logloss = log_loss(y_train, train_predprob)\n",
    "   #Print model report:\n",
    "   # print (\"logloss of train :\" )\n",
    "   # print logloss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#params = {\"objective\": \"multi:softprob\", \"eval_metric\":\"mlogloss\", \"num_class\": 9}\n",
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb1, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmcHHWd//HXp4+578zknNwJkBAghnAFFMRwBJVDIBzCShCzrIviwbq4608QV4VdVwW5jJigomSjHCJGUA4FDJADwpUQEnJOMjlmksncR3d/fn98ayadSU+mk0xPzUx/no9HPaa7qrr609U9/e76VtW3RFUxxhhjAAJ+F2CMMabvsFAwxhjTwULBGGNMBwsFY4wxHSwUjDHGdLBQMMYY08FCwZg4IvIfIvKQ33UY4xcLhX5GRPJEZKOIXB03Ll9ENovIZXHjpovI0yKyR0RqRGSViHxPRIq96deJSFRE6r1hvYj8S4prP0tEKlL5HIciUT2q+n1VvSFFz7dRRGamYtmp0FvvV39bLwOdhUI/o6r1wFzgbhEp80b/N7BcVX8PICIzgL8B/wCOUdUi4HwgApwQt7hXVTVPVfOAy4D/FpGP9M4rMYdCREJ+12DShKra0A8H4GHgUeAsoBoYFjftFeCn3Tz+OuCVTuOWAlfH3b8QeA+owYXMpLhpk7xxNd48F8ZNuwBYBdQBW4FbgFygCYgB9d4wvIvXdR/wJ+/xrwPjk1gfxwB/BXYDa4DZh1MPcDvwiPe4MYACc4AtwB7gRuAk4G3vtd8b9zzjgRe896MK+A1Q5E37tfdcTd5zfSOJdbwR+HfvuVqAkHd/q/da1gCfSLAuTgW2A8G4cZcAb3u3TwaWA7XADuBHXazTs4CKLqYVAr8CdgGbgG8BAW9aEPhfbx1sAG7y1mOoi2VtBGZ2Me0LwDrvfX2q/TMDCPBjYCew11tHU7p6v/3+f+1Pg+8F2HCYbxwUA5XeP96cuPG5QBQ4q5vHX0dcKHhfdDXAUd79o4AG4BwgDHzD++fM8O6vA/7Du3+29w94tPfYSuCjcXVO8253+SUTV8fD3hfAyd6X4G+Ahd08Jhf3pT3He8w0b70ce6j1kDgUHgSygHOBZuBJYDAwwvtSOtObf4K3vjKBMuAl4Cdxy97vy+9g6zhu/pXASCAbONp7ncPj6ksYmMCHwDlx938H3OrdfhW41rudB5zaxTK6fL9wgfAHIN+r4wPg8960G3FfyuXe+n6OwwgF73NV5b2fmcBPgZe8aecBK4AiXEBMwvth1NX7bUNygzUf9VOqugf3CzMHeDxuUjGuWXB7+wgR+W9vv0KDiHwrbt5TvfH1uK2EXwNrvWlXAH9S1b+qahvwQ9wX0wzcL9E84E5VbVXVF4Cngau8x7YBk0WkQFX3qOobh/jyHlfVpaoawYXC1G7m/xSwUVUXqGrEe77HcE1iPVHPd1W1WVX/gvsSf1RVd6rqVuBl4CMAqrrOW18tqroL+BFw5kGWe7B13O4eVd2iqk24sM/0XktYVTeq6oddLPtRvPdDRPJxv54fjVsfE0SkVFXrVfW1Q1kZIhL0av+mqtap6kbclsG13iyzgbtVtcL7nN55KMuP81lgvqq+oaotwDeB00RkjPca8nFbiKKqq1W1Mu71Hcn7ndYsFPopEbkG9wvtOeCuuEl7cM0Uw9pHqOo31O1XeAL3S7rda6papG6fwlDgWOD73rThuGaB9mXEcL9SR3jTtnjj2m3ypgFcivsS2iQifxeR0w7x5W2Pu92IC6CDGQ2c4gVcjYjU4L5QhvZQPTvibjcluJ8HICKDRWShiGwVkVrgEaD0IMs92DputyVu+jrgK7itmZ3ecw3vYtm/BT4jIpnAZ4A3VLX9uT6P20p5X0SWicinDlJjIqW4LcRNcePi3//h8XV3un0oOq+felzT3Ajvh8i9uKbGHSIyT0QKvFmP9P1OaxYK/ZCIDMa1p34B+Gdgtoh8DEBVG3Dt8J85lGWq6g7cr+tPe6O24b5s259TcM0YW71pI0Uk/vMzypuGqi5T1YtwTSxPAovan+ZQajoEW4C/ewHXPuSp6r/0cj0/8JZ5vKoWANfgmjbadX6+g63jhI9R1d+q6hne45T9fxDEz7cK94U6C7gaFxLt09aq6lW49XEX8HsRyU3+ZVKF+zU+Om5cx/uPa74pj5s28hCWHa/z+skFBrHvc3aPqp6I+zFzFPBv3viu3m+TBAuF/ule4ElVfdHbZP4G8HPvVyHe/etF5FYvQBCRcmBsVwsUkUG4nZHveaMWAZ8UkU+ISBj4Om5n5xJc6DQA3xCRsIichQuThSKSISKfFZFCr0mkFtfsAe4X9iARKeyh9dDuaeAoEbnWqycsIieJyKRericftxO5RkRG4H1JxdkBjIu7f7B1fAAROVpEzvbe52bcVko00bye3wJfBj6G26fQvpxrRKTM2zKp8UZ3uRwRyYofcFuii4DveYdDjwa+htsyan9dN4vICBEpwu0c70640/OEvPrniMhU7zV/H3hdVTd67+8p3npr8NZHtJv32yTD750aNhzaAFyM+wVV1Gn888D34u6fAizG/dPXAO8C3wMGedOvw/2ztB95sxPX5jw4bhmX4HYY7gX+jrfj1pt2rDdurzfPJd74DOAZXDNWLbAMOCPucfNxTQA1dH300X/F3T+LbnZOe/MdjTtiaZe3/Bdw+yIOqR4S72gOxc1fQdxOfNwX4bfi1skKb32uxH3JV8TNexGw2XuuW5JYxxvZf8f08bh9P3W4nfFPJ1qHcfOPwn2B/6nT+Ee897se9yPg4i4ef5b3+jsPE3D7rh7x1vcW4NvsO/oohNuSrcYdffRV3JaFdPE8GxM8x395027E7TRvf73l3vhP4I44qmffkV553b3fNnQ/iLeCjTEmJURkFvCgqo7udmbjO2s+Msb0KBHJFpELRCTkNaPdhjvIwfQDtqVg+gUR+Sjw50TT1B09ZfoIEcnBNYUdg9vv8SfgZlWt9bUwkxQLBWOMMR2s+cgYY0yHftfJVmlpqY4ZM8bvMowxpl9ZsWJFlaqWdTdfvwuFMWPGsHz5cr/LMMaYfkVENnU/lzUfGWOMiWOhYIwxpoOFgjHGmA79bp+CMcYkq62tjYqKCpqbm/0upddkZWVRXl5OOBw+rMdbKBhjBqyKigry8/MZM2YMrhPagU1Vqa6upqKigrFju+z/8qCs+cgYM2A1NzczaNCgtAgEABFh0KBBR7RlZKFgjBnQ0iUQ2h3p602bUFi/Yy+Ll76HdethjDmYK372Klf87FW/y/BN2oTCcz//JhcsnkFDY4PfpRhj0kR1dTVTp05l6tSpDB06lBEjRnTcb21tTXo58+fPZ/v27d3P2APSZkfztEnj4R2ord5BXq51qmmMSb1BgwaxcuVKAG6//Xby8vK45ZZbDnk58+fPZ9q0aQwdOrT7mY9Q2oRCOM9dP71+z04YNd7naowx6e6Xv/wl9913H62trcyYMYN7772XWCzGnDlzWLlyJarK3LlzGTJkCCtXruSKK64gOzubpUuXkpGRkbK60iYUsgpdP1BNtbt8rsQY44fv/PE9Vm3r/pIOqyrdPMnsV5g8vIDbPn3sIdfy7rvv8sQTT7BkyRJCoRBz585l4cKFjB8/nqqqKt555x0AampqKCoq4qc//Sn33nsvU6dOPeTnOlRpEwo5RS4UWvZaKBhj/PXcc8+xbNkypk+fDkBTUxMjR47kvPPOY82aNdx8881ccMEFnHvuub1eW9qEQn7xEAAi9dU+V2KM8UOyv+jbtxD+759PS1ktqsr111/Pd7/73QOmvf322/z5z3/mnnvu4bHHHmPevHkpqyORtDn6KL94MACxRgsFY4y/Zs6cyaJFi6iqqgLcUUqbN29m165dqCqXX3453/nOd3jjjTcAyM/Pp66urldqS5sthWBGFg1kEWja7Xcpxpg0d9xxx3Hbbbcxc+ZMYrEY4XCYBx98kGAwyOc//3lUFRHhrrvuAmDOnDnccMMNtqO5p9VJAcHmPX6XYYxJQ7fffvt+96+++mquvvrqA+Z78803Dxg3e/ZsZs+enarS9pNWodAQLCCjba/fZRhj+rBU7kvoD9JmnwJAU7iQ7LYav8swxpg+K61CoTWjmLxo98cpG2NMukpZKIjIfBHZKSLvdjFdROQeEVknIm+LyLRU1dIulllEvvbOHnxjjOmPUrml8DBw/kGmzwImesNc4IEU1gKAZpdQKA00Nbek+qmMMaZfSlkoqOpLwMGO/7wI+JU6rwFFIjIsVfUABHIHAVCze0cqn8YY058t+KQb0pSf+xRGAFvi7ld44w4gInNFZLmILN+16/C7qQi1d4q3e+dhL8MYY5LVE11nz5kzhzVr1qS40n38PCQ10eWBEl4BR1XnAfMApk+ffthXyckscKHQaP0fGWN6QTJdZ6sqqkogkPg3+oIFC1JeZzw/txQqgJFx98uBbal8wpwi19VFi/WUaozx0bp165gyZQo33ngj06ZNo7Kykrlz5zJ9+nSOPfZY7rjjjo55zzjjDFauXEkkEqGoqIhbb72VE044gdNOO42dO3u+1cPPLYWngJtEZCFwCrBXVStT+YR5Xv9HbXVVqXwaY0xf9OdbYfs73c+3/W33N5n9CkOPg1l3HlY5q1atYsGCBTz44IMA3HnnnZSUlBCJRPj4xz/OZZddxuTJk/d7zN69eznzzDO58847+drXvsb8+fO59dZbD+v5u5LKQ1IfBV4FjhaRChH5vIjcKCI3erMsBtYD64CfA19MVS3tCkpcT6mxegsFY4y/xo8fz0knndRx/9FHH2XatGlMmzaN1atXs2rVqgMek52dzaxZswA48cQT2bhxY4/XlbItBVW9qpvpCvxrqp4/kVB2Po1kIY0WCsaknWR/0bdvIcz5U+pqAXJzcztur127lrvvvpulS5dSVFTENddcQ3Nz8wGPie8ILxgMEolEeryutDqjGWCvFBJutlAwxvQdtbW15OfnU1BQQGVlJc8++6xvtaRVh3gA9eESMlvsmgrGmL5j2rRpTJ48mSlTpjBu3DhOP/1032oR14rTf0yfPl2XL19+2I9/54cXkNNQwfjb3u7BqowxfdHq1auZNGnSoT2ol5qPUinR6xaRFao6vbvHpt2WQltWKYV1CbtjMsaYfh0GPSHt9inEcssoppZG6//IGGMOkHahEMwrIyjK7l3W/5Ex6aC/NZEfqSN9vWkXCuHCoQDsrU7pydPGmD4gKyuL6urqtAkGVaW6upqsrKzDXkba7VPIKXKh0Lg7pSdPG2P6gPLycioqKjiSjjT7m6ysLMrLyw/78WkXCvmlrnfu1r3WfGTMQBcOhxk7dqzfZfQradd8VFjqeueO1lkoGGNMZ2kXChl5JbQRhAa7poIxxnSWdqFAIMBeKSDUZGc1G2NMZ+kXCkBd0Lq6MMaYRNIyFCoj+WRYp3jGGHOAtAyF3EHDKaMmbY5dNsaYZKVlKMTyhlFKDXVN1tWFMcbES8tQCBYNIyQxqnZs9bsUY4zpU9IyFLKK3dl+e3ds9rkSY4zpW9IyFPLLRgLQUF3hcyXGGNO3pGUoFA8dDUDbHgsFY4yJl5ahkFU0jCgCddv9LsUYY/qUtAwFgiFqpJhwo4WCMcbES89QAPaGS8luTp/udI0xJhlpGwpNmWUUtFkoGGNMvLQNhbacoQzS3USiMb9LMcaYPiNtQ0EKhlEi9VTV1PpdijHG9BlpGwp/rXAvvXq7ncBmjDHt0jYUZp99MgB7d2z0txBjjOlDUhoKInK+iKwRkXUicmuC6aNF5HkReVtE/iYih3+16UNUMnw8AM27NvbWUxpjTJ+XslAQkSBwHzALmAxcJSKTO832Q+BXqno8cAfwg1TV01le2RgAYjVbeuspjTGmz0vllsLJwDpVXa+qrcBC4KJO80wGnvduv5hgeupk5FAjhYTqrKdUY4xpl8pQGAHE/wyv8MbFewu41Lt9CZAvIoM6L0hE5orIchFZvmtXz51bUJMxlLzmyh5bnjHG9HepDAVJMK7zpc5uAc4UkTeBM4GtQOSAB6nOU9Xpqjq9rKysxwpszB5Gcdt2uwKbMcZ4UhkKFcDIuPvlwLb4GVR1m6p+RlU/AvynN25vCmvaTzR/BMOoZm9ja289pTHG9GmpDIVlwEQRGSsiGcCVwFPxM4hIqYi01/BNYH4K6zlAqGQ0OdJC5fZt3c9sjDFpIGWhoKoR4CbgWWA1sEhV3xORO0TkQm+2s4A1IvIBMAT4XqrqSSSnbCwAe7Z92JtPa4wxfVYolQtX1cXA4k7jvh13+/fA71NZw8EUD3eh0GjnKhhjDJDGZzQD5A8ZB0B09yafKzHGmL4hrUNBckpoIotArV2W0xhjIM1DARF2h4eR22ihYIwxkO6hADTkllPato1YzM5VMMaYtA+FSOEYRrKTHbVNfpdijDG+S/tQyCgbR7a0sq1io9+lGGOM79I+FAqGHwVAzdYPfK7EGGP8l/ahUFJ+NADNO+0ENmOMSftQCJWMJkoA2bPB71KMMcZ3aR8KhDLYHSwjp96u1WyMMRYKQF12OcWt26wLbWNM2rNQANoKR1Ou26mqty60jTHpzUIBCJdNZJDUsaHCutA2xqQ3CwWgcORkAKo3vetzJcYY4y8LBaBk9LEANFe+73MlxhjjLwsFQIrH0EaI0O61fpdijDG+slAACIbZnTGc/IaNfldijDG+slDwNOSPY0SkgvqWiN+lGGOMbywU2pVOZLRsZ/2OGr8rMcYY31goeHJHTCJDovzPwr/4XYoxxvjGQsEzaPQUAC4cUedzJcYY459uQ0FExotIpnf7LBH5sogUpb603hUacgwAunONz5UYY4x/ktlSeAyIisgE4BfAWOC3Ka3KD1mF7AkNJr/WDks1xqSvZEIhpqoR4BLgJ6r6VWBYasvyR13hUYyJbqS6vsXvUowxxhfJhEKbiFwFfA542hsXTl1J/pEhxzJetvLBtt1+l2KMMb5IJhTmAKcB31PVDSIyFngktWX5o2jM8WRIlG3rrQ8kY0x6CnU3g6quAr4MICLFQL6q3pnqwvyQP+oEAJoq3gHO8bcYY4zxQTJHH/1NRApEpAR4C1ggIj9KZuEicr6IrBGRdSJya4Lpo0TkRRF5U0TeFpELDv0l9KDSo4gQoH7L276WYYwxfkmm+ahQVWuBzwALVPVEYGZ3DxKRIHAfMAuYDFwlIpM7zfYtYJGqfgS4Erj/UIrvcaFMarJHM0E309wW9bUUY4zxQzKhEBKRYcBs9u1oTsbJwDpVXa+qrcBC4KJO8yhQ4N0uBHy/yk1b6WSOkU2sqqz1uxRjjOl1yYTCHcCzwIequkxExgHJHMw/AtgSd7/CGxfvduAaEakAFgNfSmK5KZU75kTKpYo1H27wuxRjjOl13YaCqv5OVY9X1X/x7q9X1UuTWLYkWlyn+1cBD6tqOXAB8GsROaAmEZkrIstFZPmuXbuSeOrDlz/uJABqNyxP6fMYY0xflMyO5nIReUJEdorIDhF5TETKk1h2BTAy7n45BzYPfR5YBKCqrwJZQGnnBanqPFWdrqrTy8rKknjqwyfD3BFI4R1vpfR5jDGmL0qm+WgB8BQwHNf880dvXHeWARNFZKyIZOB2JD/VaZ7NwCcARGQSLhRSuynQnaxC9mSNYnjjGuqa23wtxRhjelsyoVCmqgtUNeINDwPd/lz3usa4Cbc/YjXuKKP3ROQOEbnQm+3rwBdE5C3gUeA6Ve3cxNTr2oYcz5TABt6u2Ot3KcYY06u6PXkNqBKRa3Bf2uD2A1Qns3BVXYzbgRw/7ttxt1cBpydXau8pGDedrE1P86e1H3L6hANas4wxZsBKZkvhetzhqNuBSuAyXNcXA1bWaLezue7D132uxBhjelcyRx9tVtULVbVMVQer6sW4E9kGruHTiBIkf9cbRKIxv6sxxphec7hXXvtaj1bR12TkUFd0DMfF1vD+drsSmzEmfRxuKCQ6B2FACY05lamBD1mxfqffpRhjTK853FDw/QihVMsbP4McaWHrGjuJzRiTPro8+khE6kj85S9Adsoq6itGngyAbF1KLHYlgcCA3zgyxpiuQ0FV83uzkD6ncCRNWUOY0rCKVZW1TBlR6HdFxhiTcofbfDTwicCYMzg1sJol6/w9ydoYY3qLhcJBZE88kzLZy4Y1K/0uxRhjeoWFwsGMOQOA4KZ/0BKxi+4YYwY+C4WDKRlHc/YQTgms4vX1u/2uxhhjUi6ZrrPrRKS207DF6057XG8U6RsRQqKcFljFi6u3+12NMcakXDJbCj8C/g3XbXY5cAvwc9zlNeenrrS+IXTudyiVWjavXkof6MDVGGNSKplQOF9Vf6aqdapaq6rzgAtU9f+A4hTX57/xZwNwVN1S1lc1+FyMMcakVjKhEBOR2SIS8IbZcdMG/k/n/KG0lk3hzOBbXDd/qd/VGGNMSiUTCp8FrgV2esO1wDUiko27iM6Al3H0OUwPfMCwrIjfpRhjTEol03X2elX9tKqWesOnVXWdqjap6iu9UaTvJswkRJTiHf9gy+5Gv6sxxpiUSeboo3LvSKOdIrJDRB4TkfLeKK7PGHkK0axizg0uZ/E7lX5XY4wxKZNM89EC4ClgOO4IpD9649JHMEzw6FnMDLzJ/c+v9rsaY4xJmWRCoUxVF6hqxBseBspSXFffM+nTFEoDx0feYe0Ou/COMWZgSiYUqkTkGhEJesM1QHWqC+tzxn8cDecwK7iMx9/c6nc1xhiTEsmEwvXAbGA7UAlcBsxJZVF9UjgbOeo8PhVewdNvbCIWG/hH4xpj0k8yRx9tVtULVbVMVQer6sXAZ3qhtr7nuMspiNUwrn4FL6217rSNMQPP4XaI97UeraK/mDATzSpkduarPPLaZr+rMcaYHne4oZCe16YMZSKTL+IcWc6S9zeztabJ74qMMaZHHW4opG+D+vFXkhFrZJa8zmUPLPG7GmOM6VFdhkIXXWbXikgd7pyF9DR6BpSM518K/kF9S4T6Fuv6whgzcHQZCqqar6oFCYZ8VQ31ZpF9ighM+ycmNL/D4JZNLFxq+xaMMQNHSq+8JiLni8gaEVknIrcmmP5jEVnpDR+ISE0q6+kxU68GhC+HnuSuZ96nuc0u1WmMGRhSFgoiEgTuA2YBk4GrRGRy/Dyq+lVVnaqqU4GfAo+nqp4elTcYplzKJ7PeIiPayC9e2eB3RcYY0yNSuaVwMrDO62W1FXeltosOMv9VwKMprKdnnfpFQm31fGvEm9z/4jp21bX4XZExxhyxVIbCCGBL3P0Kb9wBRGQ0MBZ4oYvpc0VkuYgs37Wrj5w0Vn4iZORzWc18Wlpb+eQ9L/tdkTHGHLFUhkKicxm6OpT1SuD3qpqwcV5V56nqdFWdXlbWh/riu/h+wtFG7jrmQ6rqW/jAOsozxvRzqQyFCmBk3P1yYFsX815Jf2o6anfMp6BsEhfXPUpAlMseWGJ9Ihlj+rVUhsIyYKKIjBWRDNwX/1OdZxKRo4Fi4NUU1pIagQB87BaC1Wv4VeFD1DZH+I0domqM6cdSFgqqGsFdw/lZYDWwSFXfE5E7ROTCuFmvAhaqav/8iX3sJTDkOE7LWEdZVozb/vCuXbLTGNNvSX/7Lp4+fbouX77c7zL2t/5v8KuL2Hv6f3L6KycwZUQBv73hVAKB9OwiyhjT94jIClWd3t18KT15LW2MOwuyiylcciffnTmE19bv5qwfvuh3VcYYc8gsFHrK9c8CcHHtI5wzeQjbappZtnG3z0UZY8yhsVDoKWVHQ+5gZNnP+fGZQYIB4ap5r7GxqsHvyowxJmkWCj3pi0sgp5S8v3ydxV+aQX5WiH+av5Sddc1+V2aMMUmxUOhJOSUw6y7YuoLxC89keFE2FXsa+dz8ZdQ2t/ldnTHGdMtCoadNuRSOvgBqt/Kn2UU8POdk3q+s5bQfPM/uhla/qzPGmIOyUOhpInDhTwGFh87hY6MymPdP02lsjTLjzuftEp7GmD7NQiEVckvhmicg1ga/v55zjill4RdOJRwMcOn9S6yPJGNMn2WhkCpjPwpFo2Hdc/DXb3PKuEEs+ufTqG5oYdbdL7PcDlc1xvRBFgqp9OU34OS58Oq9sOJhJg0r4IWvn0UoIFz+4Kvc+8JaItGY31UaY0wHC4VUO+8HMGEm/PEr8PYiRpbksPQ/ZlKSm8EP//IBV8x7jU3Vdi6DMaZvsFBItWAIZv8aMgvg8S/Ae09QmBNmxf87h7uvnMrKLTV8/Id/46GX1xO1breNMT6zDvF6S2sDPHIpbFkKs38Jkz4NQOXeJmb95GVqmtrIzQwyuiSHxTd/zOdijTEDjXWI19dk5MJnfwfhHPi/a2HNMwAMK8zmzW+fwz1XfYSWthirKus44TvPsmLTHp8LNsakI9tS6G3Ne+FHk92Ww0X3wUc+2zGpsTXCeT9+iYo9TShQmB1meGEWi2/+KCLWDbcx5vDZlkJflVUIX1vt/v7hi/D3/wEvmHMyQrz872fz7nfO49ZZx9DQEmH19jqOve1ZFi3fQlNrwktYG2NMj7EtBb9EWuGpL8HbCyF3MNz8FmTk7DdLU2uUC+5+ie21LTS1uUAoy8vg3quncfLYEtt6MMYkLdktBQsFP6nC3++Cv90JQ6a4HdCDxieYTXl9w25u+u0bVNW7/pMyQwFK8zJZOPdURpbkHPAYY4yJZ6HQn6x9Dh6/AWJROO/7MPWzEEjcstfYGuGZd7dz+1PvUdscASA/K0RxTgZF2WH+cNPptgVhjDmAhUJ/U7MZHp8Lm1915zRc/wwMOfagD7n4vldYVVlHUKSjeSkzFKAoJ+wFxBlkhYO9Ub0xpo+zUOiPYjFY+Rt4+isQi8CML8OZ/w6Zed0+dMvuRq79xevUNLZR29xG+3lwBVkhCrLDFGa5rYhQ0I4tMCYdWSj0Zw3V8Nxt8OavIZgJl/4cJl3ouuVOQnNblIvufYX1VQ2EgwEavaOWAgK5mSGa26KMLslh/nUnM7Ik25qbjEkDFgoDwebX4NeXQFsjlJ8MM2+HMacf8mKq61u44mevUtcSob4lQkPLvkNbw0FBFYYVZpGbGSInI8jjXzz05zDG9G2mWpO2AAAUYklEQVQWCgNFNAIrH4HF/wbRVsguhs89DUOnHPYi26IxLrnvH9R7IVHd0Er8x0CAwpwwOeEguxtbmVCWx6IbTyMnI3Tkr8cY4wsLhYGmtRGW/gxe+K47Sun4K9z+hgSHsB6OmsZWrpz3GhuqGojGlFBQaGmLEf/pEIGACGV5GWSFg+yobeaYoQU8/sUZ1gRlTB9noTBQNe2BV34CS+4BjUH2ILh6IYw8ucefqi0a49L7l9DUFqWpNcr22mZi3uclvkPXgEBWOEhrJMaQgiyywwG27m1m4uA8fnPDKeRlhiw0jPGZhcJAV78TXv8Z/OMn7kilkafCjC/B0Rd0eY5DT1FVLrl/CWu21xJTKMoJ09wWpbYpQqJPkwigkJMZJBwM0NASQYBhRdmEgwHCAeG+z06jND+TfAsQY1LCQiFdtNTDm4/AX78N0RYIZcHZ/w+mXg05Jb1eTnNblMsfXMK6nfUoUJaXSVtUqapvIScjSCSmNLVGE4ZHOwFyMoKEQ3EBUphNOChuC6Qsj1/fcAoFWRYgxiSrT4SCiJwP3A0EgYdU9c4E88wGbgcUeEtVrz7YMi0UuhCNwKonYek82PI6IHDClTD9eig/KenDWXvL7AeXEIkpbdEY63bWM7wom7ZojMq9zahCbmaQtmjyARIMCI2tUYpzMggGhN2NrQgwoiibUEDYUtPEhLJcgiKs21XP5GEFLLpxRi+9WmP853soiEgQ+AA4B6gAlgFXqeqquHkmAouAs1V1j4gMVtWdB1uuhUIStr8LKxbA8gWgUdev0vQ5cNxsyCrwu7pD1n2AhIjGlMbWCMGAEI0pyVzELhgQggEhEo0hQF5WmGBAqGtuoyQ3g6AIVQ0uXEYWZxMMCJv3uHAJeOEiYAFj+oW+EAqnAber6nne/W8CqOoP4ub5b+ADVX0o2eVaKByClnp453fwl/9012+QAEw8F46e5fY95A32u8KUaYvGuOJnrxKNKdGY8uGuesqLc4jElG01TagqxbkZRGNKTWMbCmSFAkRVaWmLERAhegj/G8GAEIspmeEAwYDQ3OaCpjA7TECEmqZWyvIyCQSEXXUtgNuKEYGtNU0IMHpQLgGBjdWNjPeCRwR+8bmTyAgFyAwFCQfFmszMYekLoXAZcL6q3uDdvxY4RVVvipvnSdzWxOm4JqbbVfWZBMuaC8wFGDVq1ImbNm1KSc0DlipsXeECYvkCt+8BYMxH4diL3dnSAzggDtfsB5ewqrIWBcaV5hGNxdhQ1UB5cQ7RmLK1xl0MqTQ3g6gq1Q2tFGSFicaU+mYXNOFggJgqbdGe+z8TXGtgTN3tcDCAiAvC7HCQgAiNbdG4UIKapjYABudnEhBhZ10LwwqzEBEq9zYBMKo4BxHYvLuRsaW53HXpCYRDQkYwQEYo0PE3HHTB1/7X9A99IRQuB87rFAonq+qX4uZ5GmgDZgPlwMvAFFWt6Wq5tqVwhFRhx3vw6JXQUAUR94XA2I/BsZe4gMgt9bfGAUpVaYnEaInEmLNgKTGvmWvdrnoARpfkEFNl8+5GRhTnoKpU7HHvz5CCTGIKO2ubKc1zt6sbXLgXZodRhb1NbeRmhoip0tDietBNRSh11jmkQkFBENpiMTJDAQShJeLOos/JCCECja1Rd6gyUO/VWpAdRnCvoyjH3W4Ps5LcDASobnBbXCKwy+tGfki+u7+j1q0PF3ZQWdPMiOJsBKio8YKvJAfBBd/oQbmIwKbq9hA8nmBACHnNiqFAgGDQ3Q+INz7obrdHYfxG276x+49vX57fW3h9IRSSaT56EHhNVR/27j8P3Kqqy7paroVCD1KFnavgvSfhvSegeq0bP/oMmDgTJpzjemq15ooBIxZTmiNRWiMxWiMx5v5qOWu9I8XGleYSU2VDVQOjSnJQhc17GgEYXphNTJXKvc0Mzs9EgZ11LaDKoLxMNC6kinIyUHXNcgXZYVSVumZ3uHJuRhAFGlsiZGUEUXVHrAFkhAKg0BKNEQ66223RGACBgKCqHcHTv46Z3Cc+QIMBFyNRbwdYRsgdSt7+mtt7OG5ui5KdEUQQhhVm8devnXl4z90HQiGEaxr6BLAVt6P5alV9L26e83E7nz8nIqXAm8BUVa3uarkWCinSvgWx6kl49X5oa3DjgxnuKKYJ58DYj7puNozpA9qb9yYNLUCB97e7pr6jBuejKGt31DNhcB4atzU2tjQXVdhQ3cCYkhwU2FTtPuvlxe5iVVv2NDKiKBtV3P4n3NaHKmyvbQbclpuqC8bB+ZkA7PD2FQ2Juz84PxMUdta5xw3Kc9Oq6lsoyc1AFfY0ui2e+C0+cNdJUYW6lgh5mSFUlaGFWfzlq/00FLwiLgB+gttfMF9VvycidwDLVfUpcdtT/wucD0SB76nqwoMt00Khl9RWwrrnXG+tTTXuKCaAjFw4cY7bHzH6NHetaWNMn9cnQiEVLBR8EG2DimWw4WV49T5oqaVjA37o8TDmDBg9A0bNgNxBvpZqjEnMQsGkTlsTbFkKm5bAaw9Aa53rhwmg7BgYfboLidGnQ8Ewf2s1xgAWCqY3RVpg25uw6R8uKD58cV9zUygLjrvMBcSo06B4jO24NsYHFgrGP9EIbH/bBcSmJbD2WddpH0Aw7K5BffpXYMSJMOyEpC43aow5MhYKpu+IxWDXahcQFcvcEU6Rln3Twzkw6dMwbCoMnwpDj4PMfP/qNWYAslAwfVv9LtfktHWFu3hQa4O7sly7UBZMmOkCYsgUd76ENT0Zc9gsFEz/U7cdKt9yTU/b34UPnt13xjVARr4Lh6FTXFAMPQ4GT3KHyRpjDirZULCL7pq+I3+oG446b9+41gbYuRq2vwM73oW3F7kmqPYd2QCDJnghMQWGHOeCo7DctiqMOQy2pWD6H1Wo2eS2Jna86wJj3XMQad43TyDkrkY35FgoneiGQROhYLiFhUlLtqVgBi4Rt3+heAxM+tS+8c21ri+n9q2Kdx6DzUv2nUMBrvvwcDYcNQtKj4Kyo9zfkvEQzurtV2JMn2OhYAaOrAIYdaobAD59t9uqqKuEqrWuw7+qde7v6j/u60K8XSjTdQY4aII3jHd/C8shEOz912OMDywUzMAm4pqMCobDuE4dibU2QvU6qPrAC4118MEz8OHzBy4nnAPjz3YhUTpxX3DkDLLmKDOgWCiY9JWRA8OOd0M8Vajf4UKiY/gQ1j0P7z+9/7yBoDu/ovPWxaAJdlSU6ZcsFIzpTGTfkVBjzth/WjQCezd7zVBxw3tPHtgcFcyAkafEBYY3FI2CUEbvvR5jDoGFgjGHIhiCknFu4Nz9p7U2wu71XlCshdfnwbaVsPnVfd18dBC3D2TyxVAyForH7vubVdBbr8aYA9ghqcb0hsbd+7Yqdm9w18qONLuT8xIFRkYeHH2+C4niMW7romgUFIxwwWTMIbIzmo3pL5prYc8GFxZ7NrjuyCPN0NZ8YJMUQDDTHSkVyoLpc/YFRkdohHv/NZg+z0LBmIEg0gq1FVCzef9hzZ+hpY6EVytuD41jPrl/YFhopDU7ec2YgSCUEbcPI4FIK9RuTRAai+GthRw0NI6etX9gFI5052SEMlP6kkzfZqFgTH8WynA7qEvGJp4eHxp7t8CeTe7v+39KfMQUuKOmhk+DopH7B0bRaBcadub3gGahYMxA1l1oRNugdtu+0Ijf2lj91P7XvWgXzHAXRyoY4Q3D3WVX22/nD7Mmqn7MQsGYdBYMQ/FoNyQSjbhuQjoHRs3mA7s2j5c3xAuLuKDoCBBvCGen7nWZw2ahYIzpWjDkNSONhNEzDpyuCs17XXDUbnVbHbXb9t3evR4++DPEogc+NhCCskn7B0XH4AWIXYGv11koGGMOnwhkF7lh8KSu52ttgNr44IgLkI0vuRP/Eu0Ul6Dra6o9LPI7hUbBcMgutv6nepCFgjEm9TJyoXSCG7oSafG2ODptbbT/3fjK/pdsbScBd5JfR1h0bqoaATmlEAik7vUNIBYKxpi+IZS57zoZXYlGXGeFiUJj3XMuOIADtzrENYEVjPD2b8RvbXh/84bY2eJYKBhj+pNgCApHuIGTEs8Ti0FjVeJ9HLXb3Dkc8Vfpi9fRPDVs/9DIH+ZCI68MMgsGdHOVhYIxZmAJBCBvsBuGfyTxPKrQtCdxaNRuhV0fwPuL978WeAdx523klbmgyPX+5g3e/3beYNeHVT8LEAsFY0z6EYGcEjcMndL1fM21+46sqt8FDTtd81X9Lve3ZjNULIOGKhLuKA9lewExBPKHQN7QuL9DvfFD+9Q+j5SGgoicD9wNBIGHVPXOTtOvA/4H2OqNuldVH0plTcYYk7SsAjeUHX3w+aIRaKz2QsMbOm7vcMOuD2DDS+4Q3gPIvp3k+e3DEG+ro33LYyjklqb80rApCwURCQL3AecAFcAyEXlKVVd1mvX/VPWmVNVhjDEpFwy5L/H8Id3P29bkQqJuB9Rvd3/rKvcdebXrfdcNSaKmq+JxcPObPV9/nFRuKZwMrFPV9QAishC4COgcCsYYkz7C2d0fZQX7wiN+a2NUghMIe1gqQ2EEsCXufgVwSoL5LhWRjwEfAF9V1S2dZxCRucBcgFGjRqWgVGOM6WOSDY8elso9G4l2uXfeE/NHYIyqHg88B/wy0YJUdZ6qTlfV6WVlZT1cpjHGmHapDIUKYGTc/XJgW/wMqlqtqu3dMP4cODGF9RhjjOlGKkNhGTBRRMaKSAZwJfBU/AwiMizu7oXA6hTWY4wxphsp26egqhERuQl4FndI6nxVfU9E7gCWq+pTwJdF5EIgAuwGrktVPcYYY7pn12g2xpg0kOw1mvvGKXTGGGP6BAsFY4wxHSwUjDHGdOh3+xREZBew6TAfXgpU9WA5A4WtlwPZOknM1suB+ss6Ga2q3Z7o1e9C4UiIyPJkdrSkG1svB7J1kpitlwMNtHVizUfGGGM6WCgYY4zpkG6hMM/vAvooWy8HsnWSmK2XAw2odZJW+xSMMcYcXLptKRhjjDkICwVjjDEd0iYUROR8EVkjIutE5Fa/6/GLiGwUkXdEZKWILPfGlYjIX0Vkrfe32O86U01E5ovIThF5N25cwvUgzj3eZ+dtEZnmX+Wp08U6uV1Etnqfl5UickHctG9662SNiJznT9WpJyIjReRFEVktIu+JyM3e+AH5eUmLUIi7XvQsYDJwlYhM9rcqX31cVafGHVt9K/C8qk4EnvfuD3QPA+d3GtfVepgFTPSGucADvVRjb3uYA9cJwI+9z8tUVV0M4P3/XAkc6z3mfu//bCCKAF9X1UnAqcC/eq9/QH5e0iIUiLtetKq2Au3XizbORey76t0vgYt9rKVXqOpLuO7a43W1Hi4CfqXOa0BRp2uBDAhdrJOuXAQsVNUWVd0ArMP9nw04qlqpqm94t+tw130ZwQD9vKRLKCS6XvQIn2rxmwJ/EZEV3rWvAYaoaiW4fwBgsG/V+aur9ZDun5+bvGaQ+XFNi2m5TkRkDPAR4HUG6OclXUIhmetFp4vTVXUabhP3X0XkY34X1A+k8+fnAWA8MBWoBP7XG59260RE8oDHgK+oau3BZk0wrt+sm3QJhW6vF50uVHWb93cn8ARuk39H++at93enfxX6qqv1kLafH1XdoapRVY3hrqPe3kSUVutERMK4QPiNqj7ujR6Qn5d0CYVurxedDkQkV0Ty228D5wLv4tbF57zZPgf8wZ8KfdfVengK+CfvqJJTgb3tzQYDXae28Etwnxdw6+RKEckUkbG4napLe7u+3iAiAvwCWK2qP4qbNCA/Lym7RnNf0tX1on0uyw9DgCfcZ5wQ8FtVfUZElgGLROTzwGbgch9r7BUi8ihwFlAqIhXAbcCdJF4Pi4ELcDtTG4E5vV5wL+hinZwlIlNxzR8bgX8G8K63vghYhTs6519VNepH3b3gdOBa4B0RWemN+w8G6OfFurkwxhjTIV2aj4wxxiTBQsEYY0wHCwVjjDEdLBSMMcZ0sFAwxhjTwULBGGNMBwsFY5IgIlM7dRt9YU91wS4iXxGRnJ5YljFHys5TMCYJInIdMF1Vb0rBsjd6y646hMcEB/DJYsZHtqVgBhQRGeNdDOXn3gVR/iIi2V3MO15EnvF6jH1ZRI7xxl8uIu+KyFsi8pLXNcodwBXehWauEJHrROReb/6HReQB70Is60XkTK9H0dUi8nDc8z0gIsu9ur7jjfsyMBx4UURe9MZdJe5CSO+KyF1xj68XkTtE5HXgNBG5U0RWeT2Y/jA1a9SkHVW1wYYBMwBjcN0uTPXuLwKu6WLe54GJ3u1TgBe82+8AI7zbRd7f64B74x7bcR93cZqFuN4xLwJqgeNwP7pWxNVS4v0NAn8DjvfubwRKvdvDcV0mlOG6InkBuNibpsDs9mUBa9i3tV/k97q3YWAMtqVgBqINqtreR80KXFDsx+sGeQbwO68/m58B7Z2//QN4WES+gPsCT8YfVVVxgbJDVd9R17Poe3HPP1tE3gDexF2xLNHV/04C/qaqu1Q1AvwGaO/ePIrrqRNc8DQDD4nIZ3B97BhzxNKiQzyTdlribkeBRM1HAaBGVad2nqCqN4rIKcAngZVeh3DJPmes0/PHgJDXk+gtwEmqusdrVspKsJxEffG3a1ZvP4K6Th5PBj6B6/X3JuDsJOo05qBsS8GkJXUXSdkgIpdDx8XWT/Buj1fV11X120AVrm/8OiD/CJ6yAGgA9orIENxFjtrFL/t14EwRKfWueXwV8PfOC/O2dArVXTP5K7iL4BhzxGxLwaSzzwIPiMi3gDBuv8BbwP+IyETcr/bnvXGbgVu9pqYfHOoTqepbIvImrjlpPa6Jqt084M8iUqmqHxeRbwIves+/WFUTXd8iH/iDiGR58331UGsyJhE7JNUYY0wHaz4yxhjTwZqPzIAnIvfhrp4V725VXeBHPcb0ZdZ8ZIwxpoM1HxljjOlgoWCMMaaDhYIxxpgOFgrGGGM6/H80b4vSqMQLZwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c1e867da90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('nestimators_first.csv')\n",
    "        \n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.legend()\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators_first.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators_first is 220\n"
     ]
    }
   ],
   "source": [
    "print('n_estimators_first is',cvresult.shape[0]) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2、调优树深度和节点最小样本权重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_depth_weight = dict(max_depth=max_depth, min_child_weight=min_child_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.60221, std: 0.00318, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.60259, std: 0.00328, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.60249, std: 0.00324, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.58874, std: 0.00413, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.58963, std: 0.00368, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.58945, std: 0.00342, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.58914, std: 0.00317, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.58937, std: 0.00370, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.58880, std: 0.00357, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.60226, std: 0.00388, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.59922, std: 0.00566, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.59578, std: 0.00374, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 5, 'min_child_weight': 1},\n",
       " 0.5887351903261084)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2 = GridSearchCV(xgb2, param_grid =param_depth_weight, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2.fit(X_train , y_train)\n",
    "\n",
    "gsearch2.grid_scores_, gsearch2.best_params_,  -gsearch2.best_score_\n",
    "#gsearch2_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'test_scores' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-13-bbccd53d4177>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mtest_scores\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'test_scores' is not defined"
     ]
    }
   ],
   "source": [
    "test_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.588735 using {'max_depth': 5, 'min_child_weight': 1}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VPW9//HXZ5bsYV9kUVGUCrJECAGVIChBUGvrxbpdbbGirV6vrfenVVtvrV5t9XqvVVv1VnGrdau2erEXNcENXCEoKIIKKkoE2RTIQpKZzOf3xzmZnIRJMmQymSyf5+ORBzPnfM853zkh5z3fs3y/oqoYY4wxbeVLdQWMMcZ0bRYkxhhjEmJBYowxJiEWJMYYYxJiQWKMMSYhFiTGGGMSYkFijDEmIRYkxhhjEmJBYowxJiGBVFegIwwYMEBHjBiR6moYY0yXsnLlyh2qOrC1cj0iSEaMGEFpaWmqq2GMMV2KiHwRTzk7tWWMMSYhFiTGGGMSYkFijDEmIRYkxhhjEmJBYowxJiEWJMYYYxJiQWKMMSYhPeI5EmM6SiSiVIfrqA5FqA7VuT8Rd1odNfXT3TLhuggigt8n+AR8Is6Pr+F1/TwRwd9knrfsPuX2WU/jeSK4yzQpJ4L4iF3OXb8xXhYkpttSVUJ1uu9B3HNgb3TAD0eoCTWZ3igUItQ0Xc4zvyYUobYukuqPnXTiBp6/Scg0DpyGYPT7YpSLBp0nQN3XsYOuSdDGsd3o+ppu19dMOfd1h9TPF//niHf/+ZvUXTow9C1ITIeJRJSacKwDdMPBvSbGwd174K5p9tu+JwTCDfMj2ra6+gQygn7nJ+AjI+gnPegnI+gjI+BnQE6gYX7QR3qg4bV3mYygn/To8vXzGsoF/T5UlYhCnSqRiKL1r933EYWIKnVN5qkqdRH2LddkXqNy0fXSTLmG9UTq6xWJUc6d52y3STn3faP1ecs12VbM7br1C9VFGpdz53nrV7/u+nnN1a9RuSbb7a58AsWXH8dhg3KSuh0Lkh6qpW/rjQ7WTQ7cNeEmB/EmB+6aFr7t14bb/m09ze9rOBC7B/P61znpAfpnew7iTeZHQ8BzcM+IcVD3HuiDfrFTOD1Io8BxQ6rFoPMGUytB1xDETcolK4gjjcv1y05L+v6zIOkkutO39X7ZaY0O0NGDdNNv7Z5psb6te7/t+312UDfJ4/MJPuz/WFtZkLTgq9/dQvWmMmTK0dTlT6Gmz4AWzp/bt3VjTM+U1CARkTnAHYAfWKiqN8cocwbwG0CB1ap6jjv9R8C1brEbVfVhEckCngJGAnXAc6p6dbLq//wnO/nOeysY8PISAL7oNYTSwUdQOvgI1vYbQZ3Pv88yIuz7bTzQcBC3b+vGmO5GVJNzpUlE/MAnQBFQBqwAzlbVtZ4yhwN/BY5X1W9FZJCqbhORfkApkI8TMCuBSUANMEVVXxGRNOAl4Leq+nxLdcnPz9e2dCNfsnYrX+/eS++vv6TvB6XkvL+C9HUfIHV1aFY2vvwC0o45lsxphWQNG0JG0Eea32ff1o0x3YKIrFTV/NbKJbNFUgBsUNXP3Ao9AXwPWOspcyFwl6p+C6Cq29zpJwIlqvqNu2wJMEdVHwdeccvWisi7wPBkfYCiMYPdVyPgtOkA1FVUUPnWW1QuXUbFsmXULH2FGmDvqFHkTC8ku3A6WROPQoLBZFXLGGM6lWQGyTBgk+d9GTClSZlRACLyBs7pr9+o6gvNLDvMu6CI9AG+i3PqrMP4c3LoVVREr6IiVJWa9eupXLaMiqXL2Pnwn9m58H582dlkH3M02YWF5EyfTvCAAzqyisYY06GSGSSxzu80PY8WAA4HZuC0LJaJyNjWlhWRAPA4cGd9i2efjYtcBFwEcNBBB+1v3eMiImSMGkXGqFH0v+AC6ioqqXr7LSrc1kp5iXNtJf3ww8meXkhOfWslLfm34xljTEdJZpCUAQd63g8HNsco87aqhoDPReRjnGApwwkX77Kvet7fC6xX1dub27iq3uuWIz8/v0MeOfLnZJM7axa5s2ahqtRu2BANlW/+/Ajf3P8Avqwsso45mpzC6eRMLyQ4ZEhHVM0YY5ImmRfbAzgX208AvsK52H6Oqn7oKTMH5wL8j0RkAPAekEfDBfaJbtF3gUmq+o2I3AiMBn6gqnHdM9vWi+3tqa6ikqp33naDZSnhzVsASD/8MLLdUMmaONFaK8aYTiPei+1JCxK3EicBt+Nc/3hAVW8SkRuAUlVdJM7tTf8NzMG5nfcmVX3CXfbHwC/dVd2kqg+KyHCcaycf4dzBBfBHVV3YUj06Q5B4qSq1n34aDZWq0pUQCjmtlaOPJqew0GmtDB2a6qoaY3qwThEknUVnC5KmIpWVVL7zDhVLl1K5dBmhzc4ZwLTDRkZPgWVOmoTPWivGmA5kQeLR2YPES1Wp/ewzKpYuo3LZUqpWlKKhEJKVRfbUqeRMLySnsJDgsGGtr8wYYxJgQeLRlYKkKae1spyKZW5r5auvAEgbOTJ6CiwzP99aK8aYdmdB4tGVg8RLVan9/PPoKbCqFSsaWitTpkQfiEwbbq0VY0ziLEg8ukuQNBWpqqLynXeiD0SGysoASDv0UHIKC8meXkjW5MnWWjHGtIkFiUd3DRIvp7WykcplS6mob63U1iKZmWRPmeI8EDl9OmnDk9ajjDGmm7Eg8egJQdJUZO9ep7XiPhAZ2uT0OJM2YkTDU/YFk/Glp6e4psaYzsqCxKMnBomXqlK7cWP0FFjV8uVOayUjg6wpBdFbjNOS1JWMMaZrsiDx6OlB0lRk716qli+Pdt8S+vJLANIOPpjs6dPJmW6tFWOMBUkjFiQtq924MRoqVcuXozU11loxxliQeFmQxC9SXe1prSwl9IXTWgkefFA0VLIKCvBlZKS4psaYZLMg8bAgabvaL75o6BPsHbe1kp5OVkFB9IHItBEjUl1NY0wSWJB4WJC0j0h1NVUrVjjdtyxdSu0XXwAQPOigaKhkFRTgy8xMcU2NMe3BgsTDgiQ5ar/8snFrpbraaa1Mnuw+Ze+0VmwMe2O6JgsSDwuS5IvU1FC1fEW0T7DajRsBCB54YPQp++wpU6y1YkwXYkHiYUHS8Wo3bYr2CVb5zjtOayUtzdNamU7aIdZaMaYzsyDxsCBJrUhNDVUrSqPdt9R+/jkAweHDo6fAsqdMwZeVleKaGmO8LEg8LEg6l9qysuhT9pVvv43u3eu0VvLzG/oEO+QQa60Yk2IWJB4WJJ1XpLaWvaWl0Qciaz/9FIDgsGHRPsGyp1prxZhUsCDxsCDpOmrLvqLydU9rpaoKCQbJmpxPdv1T9oceaq0VYzqABYmHBUnX1GxrZejQ6Cmw7ClT8GVnp7imxnRPnSJIRGQOcAfgBxaq6s0xypwB/AZQYLWqnuNO/xFwrVvsRlV92J0+CXgIyAQWAz/TVj6EBUn3EPrqKyqWvU7FsmVUvvVWtLWSmT+poU+wkSOttWJMO0l5kIiIH/gEKALKgBXA2aq61lPmcOCvwPGq+q2IDFLVbSLSDygF8nECZiUwyS2zHPgZ8DZOkNypqs+3VBcLku5Ha2upevdd5xTYsqXUrN8AQGDokGioZE+daq0VYxIQb5AEkliHAmCDqn7mVugJ4HvAWk+ZC4G7VPVbAFXd5k4/EShR1W/cZUuAOSLyKtBLVd9yp/8Z+D7QYpCY7kfS0sieOpXsqVPhF1cS2rzZba0sZc9zz7HrySchGCRr0qSGPsEOO8xaK8YkQTKDZBiwyfO+DJjSpMwoABF5A+f0129U9YVmlh3m/pTFmL4PEbkIuAjgIOsCvdsLDh1K3zPPoO+ZZ7itlfeiT9lvu/VWtt16K4EhQxr6BJt6NP4ca60Y0x6SGSSxvvo1PY8WAA4HZgDDgWUiMraFZeNZpzNR9V7gXnBObcVXZdMdOK2VKWRPnQJXXkloyxbnusqyZez5v/9j11//6rRWJk6MPhCZfvjh1loxpo2SGSRlwIGe98OBzTHKvK2qIeBzEfkYJ1jKcMLFu+yr7vThrazTmEaCQ4bQ94wz6HuG21p5b1X0Kfttt/4X3PpfBA44oKFPsKOPxp+Tk+pqG9NlJPNiewDnYvsJwFc4F9vPUdUPPWXm4FyA/5GIDADeA/JouMA+0S36Ls7F9m9EZAXwr8A7OBfb/6Cqi1uqi11sN80Jff2101pZuozKN98kUlkJgYCntTKd9FHWWjE9U8rv2nIrcRJwO871jwdU9SYRuQEoVdVF4vx1/jcwB6gDblLVJ9xlfwz80l3VTar6oDs9n4bbf58H/tVu/zXtQUMhqt57L9p9S83HHwO4rZVpTp9gxxxjrRXTY3SKIOksLEhMW4S2bm3oE+zNN4lUVDitlaOOij4QmT5qlLVWTLdlQeJhQWISpaEQe1etij5lX/PRRwAEBg8mu3Ca0yfYMUfjz81NcU2NaT8WJB4WJKa9hbZua+gT7M03iZSXO62VvDyypzsPRKZ/5zvWWjFdmgWJhwWJSSYNhdi7enVDa2XdOgACgwa5rRX32kqvXimuqTH7x4LEw4LEdKTQtm1U1vcJ9sYbTmvF7yczLy/6QGT66NHWWjGdngWJhwWJSRUNhz2tlaXUrHVaK/6BA8iZVkjOcceRUzjN+gQznZIFiYcFieksQtu2Ufn6G073LW+8SWTPHiQ9nezCafQqKiJn5kw7BWY6DQsSDwsS0xlpOEzVyncpLymhvKSE8NatEAySPXUquUWzyJ01i0C/fqmupunBLEg8LEhMZ6eRCNXvv8+e4hLKi4sJlZWBz0dWfj65RUXkzi4iOHhwqqtpehgLEg8LEtOVqCo1H33EnuJiyktKqN3gjAyZOWECubNnkzu7iLQDD2xlLcYkzoLEw4LEdGU1n31GeXEx5cUlVK91hvNJHz2aXrOLyC0qIv2ww1JcQ9NdWZB4WJCY7qK2rIzyYueayt733gMg7dBDyXVDJWPMGLut2LQbCxIPCxLTHYW2bqN8SQnlxSVUrVgBkQjB4cOj11QyJ0xAfL5UV9N0YRYkHhYkprsLf/stFS+9xJ7iYirfehtCIQKDBpE7axa5s2eTlT8JCSRz+CHTHVmQeFiQmJ6krrycildfpby4mIplr6PV1fj79iXnhOPpNXs22VOnImlpqa6m6QIsSDwsSExPFamqomLZ606ovPoqkcpKfDk55MycSe7sInKmTcOXmZnqappOyoLEw4LEGIjU1lL55puUF5dQ8dJL1O3ejWRmklNYSO7s2eTMOM4G7TKNWJB4WJAY05iGQlSVljrPqixZQt32HUgwSPYxxzihcvxMAn37prqaJsUsSDwsSIxpnkYi7F21ivIXi9lTUkx48xbw+8meUuDcATZrFoGBA1NdTZMCFiQeFiTGxEdVqf5wrfsAZDG1GzeCCJkTJ5JbNIteRUUEhw1LdTVNB+kUQSIic4A7AD+wUFVvbjJ/PnAr8JU76Y+qutCddwtwsjv9P1T1SXf6Ce4yPqACmK+qG1qqhwWJMftPValZv97pVLK4hJqPPwYgY+zY6LMq6YcckuJammRKeZCIiB/4BCgCyoAVwNmqutZTZj6Qr6qXNln2ZODnwFwgHXgNOF5V94jIJ8D3VHWdiFwCFKjq/JbqYkFiTOJqv/iC8pIS9hSXUP3++wCkH364EyonziZ91Ch7qr6biTdIkvmEUgGwQVU/cyv0BPA9YG2LSznGAK+pahgIi8hqYA7wV0CB+gEbegOb27vixph9pR18MP0XLKD/ggWEtmyhvGQJ5cXF7LjnHnbcfTfBgw+i1+zZTlct48ZZqPQgyQySYcAmz/syYEqMcvNEZDpO6+VyVd0ErAauE5HbgCxgJg0BtABYLCJ7gT3A1CTV3xjTjOCQIfT74Xn0++F5hHfsoHzJS5SXlLDzwYfYed9CAkOGRK+pZE6ciPj9qa6ySaJkBkmsryNNz6M9BzyuqjUi8lPgYZxTWMUiMhl4E9gOvAWE3WUuB05S1XdE5ErgNpxwabxxkYuAiwAOOuigfSoSCoUoKyujurq6TR/OdC0ZGRkMHz6cYDCY6qp0O4EBA+h71pn0PetM6nbtovyVVykvKWHXE0/y7Z8fwT9gALknnEDu7CKyCwoQ+x10O8m8RnI08BtVPdF9fw2Aqv6umfJ+4BtV7R1j3mPAX3Cus7ytqiPd6QcBL6jqmJbqEusayeeff05ubi79+/e3Jng3p6rs3LmT8vJyDrGLwx2mrqKSyqWvsae4hIqlS9GqKny9e5M7cya5s2eTfewx+NLTU11N04LOcI1kBXC4iByCc1fWWcA53gIiMkRVt7hvTwXWudP9QB9V3Ski44HxQLFbrreIjFLV+gv569pSuerqakaMGGEh0gOICP3792f79u2prkqP4s/JptdJJ9HrpJOIVFdT+cYbzm3FL73E7mefxZeVRc6M45wHIAsL8WVnp7rKpo2SFiSqGhaRS4EXcW7/fUBVPxSRG4BSVV0EXCYip+KctvoGmO8uHgSWuQf5PcC57oV3RORC4G8iEgG+BX7c1jpaiPQc9rtOLV9GhnN664QT0NpaKt9ZHg2VPYufR9LTyZ42jV6zi8iZMQN/731OTJhOrMc+kLhu3TpGjx6dohqZVLDfeeejdXVUrVwZHawrvHUrBAJkT53qDNY1axaBfv1SXc0eK95TWzbqTYrs2rWLu+++u03L3n777VRVVbVzjdrHjBkzaOszO88++yxr1zbcHR7PuqqrqykoKGDChAkceeSRXHfddW3atkkN8fvJLijggGt/xWGvvMyIJ5+g349+SO0XX/D1r69j/bRCvjjvh3zzyF8Ibd2a6uqaZliQpEh3DZJENA2SeKSnp/Pyyy+zevVqVq1axQsvvMDbb7+dpBqaZBKfj8wJExh85ZWMLH6RQ559hgE//Qnhb79h6003seG4GWw88yx23n8/tZs2tb5C02FsyDTg+uc+ZO3mPe26zjFDe3Hdd49sdv7VV1/Np59+Sl5eHkVFRQwaNIi//vWv1NTUcNppp3H99ddTWVnJGWecQVlZGXV1dfz7v/87W7duZfPmzcycOZMBAwbwyiuvxFx/Tk4O//Iv/8KSJUvo27cvv/3tb/nFL37Bl19+ye23386pp57Kxo0bOe+886isrATgj3/8I8cccwzPPPMMd911FyUlJXz99dccd9xxLF26lAMOOGCf7ezdu5fzzz+ftWvXMnr0aPbu3RudV1xczHXXXUdNTQ0jR47kwQcfJCcnhxEjRnDmmWdG6/7YY4+xbds2Fi1axGuvvcaNN97I3/72NwCeeuopLrnkEnbt2sX9999PYWFho+2LCDlu1+ehUIhQKGTXQ7oBESHjiCPIOOIIBl52GTWffeac/iouZtut/8W2W/+L9NGjnWdVZs8m/bDDUl3lHs1aJCly8803M3LkSFatWkVRURHr169n+fLlrFq1ipUrV7J06VJeeOEFhg4dyurVq1mzZg1z5szhsssuY+jQobzyyivNhghAZWUlM2bMYOXKleTm5nLttddSUlLCM888w69//WsABg0aRElJCe+++y5PPvkkl112GQCnnXYaBxxwAHfddRcXXngh119/fcwQAbjnnnvIysri/fff51e/+hUrV64EYMeOHdx4440sWbKEd999l/z8fG677bbocr169WL58uVceuml/PznP+eYY47h1FNP5dZbb2XVqlWMHDkSgHA4zPLly7n99tu5/vrrAdi8eTMnnXRSdF11dXXk5eUxaNAgioqKmDIl1nOvpitLP/RQBvz0Jxzy978xcskSBl11Fb6MDHbc+Qc+O+W7fHrSyWz7/e3s/fBDesJ1387GWiTQYsuhIxQXF1NcXMxRRx0FQEVFBevXr6ewsJArrriCq666ilNOOWWfb+MtSUtLY86cOQCMGzeO9PR0gsEg48aNY+PGjYDzDf7SSy9l1apV+P1+Pvnkk+jyf/jDHxg7dixTp07l7LPPbnY7S5cujQbQ+PHjGT9+PABvv/02a9eu5dhjjwWgtraWo48+Orpc/TrPPvtsLr/88mbX/0//9E8ATJo0KVrvoUOHsnjx4mgZv9/PqlWr2LVrF6eddhpr1qxh7Nixce0n0/WkDR9G//Pn0//8+YS2bqP8pSWUF5ewc+FCdv7pTwSHDXM7lZxNZt4ExGffl5PNgqQTUFWuueYafvKTn+wzb+XKlSxevJhrrrmG2bNnR1sTrQkGg9FTPD6fj3T3wS+fz0c47HQS8Pvf/57BgwezevVqIpEIGRkZ0eW/+uorfD4fW7duJRKJ4GvhjzHWqSRVpaioiMcff7zVZVo6FVVfb7/fH613c/r06cOMGTN44YUXLEh6iODgQfQ75xz6nXMO4W+/peLll9lTXMw3jz7KNw89RGDQIHJnzSJ3dhFZ+flIwA55yWBRnSK5ubmUl5cDcOKJJ/LAAw9QUVEBOAfxbdu2sXnzZrKysjj33HO54oorePfdd/dZNhG7d+9myJAh+Hw+HnnkEerq6gDndNL555/PY489xujRoxudkmpq+vTpPProowCsWbOG991eYadOncobb7zBhg1OD/9VVVWNWjxPPvlk9N/6lkpbPtf27dvZtWsX4FyvWbJkCUccccR+rcN0D4G+fekzbx4H/elPjHrzDYbeeiuZEyaw6+9/58v557O+cDqbr72WitdeI1Jbm+rqdisWzynSv39/jj32WMaOHcvcuXM555xzogfUnJwc/vKXv7BhwwauvPJKfD4fwWCQe+65B4CLLrqIuXPnMmTIkBavk7TmkksuYd68eTz11FPMnDmTbPfJ4t/+9rcUFhZSWFhIXl4ekydP5uSTT475DMbFF1/M+eefz/jx48nLy6OgoACAgQMH8tBDD3H22WdTU1MDwI033sioUaMAqKmpYcqUKUQikWir5ayzzuLCCy/kzjvv5Omnn2623ps3b2bBggUsXryYLVu28KMf/Yi6ujoikQhnnHEGp5xySpv3ieke/Lm59P7uKfT+7ilEqqqoeP1152L98y+w++m/4cvJIWfmTHJnF5EzbRq+zMxUV7lLa/WBRBEZCZS5HSvOwOmu5M+quqsD6tcu7IHEzmXEiBGUlpYyYMCADt2u/c5NpLaWyjffpLykhIqXXqZu1y4kM5OcwkKnq5YZx+F37wI07dvX1t+AfBE5DLgfWAQ8BpzU4lLGGNPJ+NLSyJ0xg9wZM9Drw1StWOGMAOmOrSLBINnHHOOEyvEzCfTtm+oqdwnxBEnE7TfrNOB2Vf2DiLyX7IqZ+EyZMiV66qjeI488wrhx49p1Oy+++CJXXXVVo2mHHHIIzzzzzH6vq/7uK2NSSQIBso8+muyjj2bwtdeyd9Wq6LMqFa+9Bn4/WQWTncG6Zs0iMHBgqqvcacVzausd4HbgV8B3VfVzEVmjql3mthg7tWXAfucmPqpK9Ydr3bHqi6n9/HMQIfOoo8idXUSvoiKCw4alupodoj1PbZ0P/BS4yQ2RQ3DGBjHGmG5HRMgceySZY49k4M9/Ru2GDewpLqa8ZAnbbr6FbTffQsaRR5I7eza5s4tItzFu9q/3XxHpCxyoqu8nr0rtz1okBux3bhJX+8UXlJeUsKekhOrVzmEw/fDDyC2aTe6Js0kfNapbddHTbi0SEXkVZ9CpALAK2C4ir6nqvyVcS2OM6ULSDj6Y/gsW0H/BAkJbtkQv0u+45x523H03wYMPopf7VH3GuHHdKlRaEs+prd6qukdEFgAPqup1ItKlWiTGGNPegkOG0O+H59Hvh+cR3rGD8pdeprykhJ0PPczOhfcTGDLE6VSyqIjMiRMRvz/VVU6aeJ5sD4jIEOAM4B9Jrk+P0V27ke/o8UjAeS5l3Lhx5OXlkZ/faivcmHYXGDCAvmeewUEL72PUG68z5ObfkTF6NLueeJIvzvsh66cfx5brfkPF62+goVCqq9vu4gmSG3CGy/1UVVeIyKHA+uRWq/vrrkGSiLaMR1LvlVdeYdWqVW0OMWPai793b/p8//scePddjHrrTYb9/jayCiaz+7nn2LRgAZ9MK2Tz1ddQ/vLLRJrcut9VtXpqS1WfAp7yvP8MmJfMSnW456+Grz9o33UeMA7m3tzsbBuPpH3GIzGmM/NlZ9Nr7lx6zZ1LpLraear+xWLKX36Z3c8+iy8ri5wZx5FbVETO9On43G6KuppWWyQiMlxEnhGRbSKyVUT+JiLDO6Jy3ZmNR9J+45GICLNnz2bSpEnce++9CfxWjEkeX0YGuccfz9BbbmbU68s4cOFCep1yCpXvLOery/+NT445lk3/cim7//d/qdu9O9XV3S/xXGx/EKdLlB+47891pxW1tqCIzAHuAPzAQlW9ucn8+cCtwFfupD+q6kJ33i3Aye70/1DVJ93pAtzo1qcOuEdV74zjczSvhZZDR7DxSBIbj+SNN95g6NChbNu2jaKiIo444gimT58e134yJhUkLY2caceSM+1YDrju1+x99132FJe4fYC9BIEA2VOnkju7iNwTTiDQv3+qq9yieIJkoKo+6Hn/kIj8vLWFRMQP3IUTOGXAChFZpKpNT4I/qaqXNln2ZGAikAekA6+JyPOqugeYDxwIHKGqEREZFMdn6NRsPJLExiMZOnQo4LSwTjvtNJYvX25BYroM8fvJmjyZrMmTGXzN1VR/8IHzrEpxCV//+jq+/s31ZE2a5DwAWTSLYDNnB1IpnovtO0TkXBHxuz/nAjvjWK4A2KCqn6lqLfAE8L046zUGeE1Vw6paCawG5rjzLgZuUNUIgKpui3OdnYqNR9I+45FUVlZGl6msrKS4uNgGtTJdlvh8ZE6YwKArrmDkiy9wyLPPMOCnP6Vu17dsvekmNsyYyednnsnO+++ndtOmVFc3Kp4g+THOrb9fA1uA03G6TWnNMMD7ScvcaU3NE5H3ReRpETnQnbYamCsiWSIyAJiJ0woBGAmcKSKlIvK8iBwea+MicpFbpnT79u1xVLdjeccjKSkpiY5HMm7cOE4//XTKy8v54IMPKCgoIC8vj5tuuolrr70WaBiPZObMmQnV4ZJLLuHhhx9m6tSpfPLJJzHHI7nttttYuHAh69ati7mOiy++mIqKCsaPH89//ud/xhyPZPz48UydOpXpue6kAAAa0UlEQVSPPvooulz9eCR33HEHv//97wFnPJJbb72Vo446ik8//bTZenuvkWzdupVp06YxYcIECgoKOPnkk6On9IzpykSEjCOOYOBl/8qhzz3HoYsXM/DyyyFcx7Zb/4tPi2bz2fdPY/vdd1PjfmFLWV33p4uU6EIiP1fV21sp8wPgRFVd4L4/DyhQ1X/1lOkPVLhjnfwUOENVj3fn/QrnOsh2YBuwXFXvEJEK4DpV/W8R+SfgclVt8eKBdZHSudh4JMYkprbsK8qXlFBeXMLe994DVdIOPdQdq76IjDFj2uWp+ni7SGnrULvxdI9SRkMrAmA4sNlbQFV3qmr9jdT3AZM8825S1TxVLQKEhmdXynDGSAF4BmegLWOM6THShg+j//z5jHjsUQ577VUG//rfCQwexM6FC9k473Q+nVXE1ptvoerd99BIJOn1aetQu/FE3QrgcLe34K+As4BzGq1EZIiqbnHfngqsc6f7gT6qulNExuOERbFb7lngeOAB4DjgE3owG4/EmJ4tOGgQ/c45h37nnEP422+pePllyotL+PbRR/nmoYcY8dRTZI5L7nXDtgZJq+fD3MGwLsV5Kt4PPKCqH4rIDUCpqi4CLhORU4Ew8A3OHVkAQWCZ2zTbA5yrqvW37NwMPCoilwMVwII2foZu4Z133umQ7Zx44omceOKJHbItY0zbBPr2pc+8efSZN4+68nIq33iDjLFHJn+7zc0QkXJiB4YAmfGsXFUXA4ubTPu15/U1wDUxlqvGuXMr1jp30fB8iTHGmBj8ubn06qAbT5oNElXN7ZAaGGOM6dLaerHdGGOMASxIjDHGJMiCJEW6azfyHT0eyccff0xeXl70p1evXtx+e4uPOBlj2pkFSYp01yBJRFvGI/nOd77DqlWror0mZ2VlcdpppyWphsaYWOIZsz3W3Vu7gVLg/7njk3Rptyy/hY+++aj1gvvhiH5HcFXBVc3Ot/FI2n88kpdeeomRI0dy8MEHt/4LMsa0m3haJLcBV+L0kzUcuALnKfQncB4KNG1g45G033gk9Z544okWu7w3xiRHPA8kzlHVKZ7394rI26p6g4j8MlkV60gttRw6go1Hkth4JPXrX7RoEb/73e9a3TfGmPYVT5BEROQM4Gn3/emeefvf46PZh41Hkth4JADPP/88EydOZPDgwc2WMcYkRzyntv4ZOA+nB95t7utzRSQTuLSlBU3zbDyS9hmPpN7jjz9up7WMSZFWWyTuxfTvNjP79fatTs/hHY9k7ty50fFIwLlQ/pe//IUNGzZw5ZVX4vP5CAaD3HPPPUDDeCRDhgxp8TpJay655BLmzZvHU089xcyZM2OOR5KXl8fkyZM5+eSTY3bBfvHFF3P++eczfvx48vLyYo5HUt+p5I033sioUaOAhvFIIpFItNVy1llnceGFF3LnnXfy9NNP77Oteps3b2bBggXR01tVVVWUlJTwpz/9qc37whjTdq2ORyIiw4E/AMfinMp6HfiZqpYlv3rtw8Yj6VxsPBJjuob2HI/kQWARMBTnzq3n3GnGGGNMXBfbB6qqNzgeEpGfJ6tCZv/YeCTGmFSLJ0h2iMi5QP3tN2cDO5NXJbM/bDwSY0yqxXNq68fAGcDXwBac23/PT2aljDHGdB2tBomqfqmqp6rqQFUdpKrfB/6pA+pmjDGmC2hrp43/1q61MMYY02W1NUiafxTZGGNMj9LWIImraxQRmSMiH4vIBhG5Osb8+SKyXURWuT8LPPNuEZE17s+ZMZb9g4hUtLH+Kdddu5Hv6PFIAO644w7Gjh3LkUceaWORGJMCzQaJiJSLyJ4YP+U4z5S0SET8wF3AXGAMcLaIjIlR9ElVzXN/FrrLngxMBPKAKcCVItLLs+58oM9+fM5Op7sGSSLaMh7JmjVruO+++1i+fDmrV6/mH//4B+vXr09SDY0xsTR7+6+q5ia47gJgQ/14JSLyBPA9IJ4jxRjgNVUNA2ERWQ3MAf7qBtStwDlAu4xg9PVvf0vNuvYdjyR99BEc8MvmO0e28UjaZzySdevWMXXqVLKysgA47rjjeOaZZ/jFL36xH78tY0wikjlC4jBgk+d9mTutqXki8r6IPC0iB7rTVgNzRSRLRAYAM4H6eZcCi1R1S7Iq3hFsPJL2GY9k7NixLF26lJ07d1JVVcXixYvZtGkTxpiOE88DiW0V64J802srzwGPq2qNiPwUeBg4XlWLRWQy8CawHXgLp2UyFPgBMKPVjYtcBFwEcNBBB7VYtqWWQ0ew8UjaPh7J6NGjueqqqygqKiInJ4cJEyYQCCTzv7Uxpqlk/sWV0dCKAGd0xc3eAqrqfUL+PuAWz7ybgJsAROQxYD1wFHAYsMEdwyJLRDao6mFNN66q9wL3gtNpYzt8nqSx8UgSG4/kggsu4IILLgDgl7/8JcOHD292fcaY9pfMU1srgMNF5BARSQPOwun8MUpEhnjengqsc6f7RaS/+3o8MB4oVtX/U9UDVHWEqo4AqmKFSFdg45G033gk27ZtA+DLL7/k73//u41LYkwHS1qLRFXDInIp8CLgBx5Q1Q9F5AagVFUXAZeJyKlAGPgGmO8uHgSWud9U9wDnuhfeuw0bj6T9xiOZN28eO3fuJBgMctddd9G3b9827xNjzP5rdTyS7sDGI+lcbDwSY7qG9hyPxBhjjGmW3d7Sxdl4JMaYVOvRQaKqLd4x1BXYeCTx6QmncI1JlR57aisjI4OdO3faAaYHUFV27tzZ6PZmY0z76bEtkuHDh1NWVsb27dtTXRXTATIyMuz5EmOSpMcGSTAY5JBDDkl1NYwxpsvrsae2jDHGtA8LEmOMMQmxIDHGGJMQCxJjjDEJsSAxxhiTEAsSY4wxCbEgMcYYkxALEmOMMQmxIDHGGJMQCxJjjDEJsSAxxhiTEAsSY4wxCbEgMcYYkxALEmOMMQlJapCIyBwR+VhENojI1THmzxeR7SKyyv1Z4Jl3i4iscX/O9Ex/1F3nGhF5QESCyfwMxhhjWpa0IBERP3AXMBcYA5wtImNiFH1SVfPcn4XusicDE4E8YApwpYj0css/ChwBjAMygQUx1mmMMaaDJLNFUgBsUNXPVLUWeAL4XpzLjgFeU9WwqlYCq4E5AKq6WF3AcsCGvTPGmBRKZpAMAzZ53pe505qaJyLvi8jTInKgO201MFdEskRkADATONC7kHtK6zzghVgbF5GLRKRUREptOF1jjEmeZAaJxJimTd4/B4xQ1fHAEuBhAFUtBhYDbwKPA28B4SbL3g0sVdVlsTauqveqar6q5g8cOLDtn8IYY0yLkhkkZTRuRQwHNnsLqOpOVa1x394HTPLMu8m9blKEE0rr6+eJyHXAQODfklR3Y4wxcUpmkKwADheRQ0QkDTgLWOQtICJDPG9PBda50/0i0t99PR4YDxS77xcAJwJnq2okifU3xhgTh0CyVqyqYRG5FHgR8AMPqOqHInIDUKqqi4DLRORUnNNW3wDz3cWDwDIRAdgDnKuq9ae2/gf4AnjLnf93Vb0hWZ/DGGNMy8S5+al7y8/P19LS0lRXwxhjuhQRWamq+a2VsyfbjTHGJMSCxBhjTEIsSIwxxiTEgsQYY0xCLEiMMcYkxILEGGNMQixIjDHGJMSCxBhjTEIsSIwxxiTEgsQYY0xCLEiMMcYkxILEGGNMQixIjDHGJMSCxBhjTEIsSIwxxiTEgsQYY0xCLEiMMcYkxILEGGNMQixIjDHGJCSpQSIic0TkYxHZICJXx5g/X0S2i8gq92eBZ94tIrLG/TnTM/0QEXlHRNaLyJMikpbMz2CMMaZlSQsSEfEDdwFzgTHA2SIyJkbRJ1U1z/1Z6C57MjARyAOmAFeKSC+3/C3A71X1cOBb4IJkfQZjjDGtS2aLpADYoKqfqWot8ATwvTiXHQO8pqphVa0EVgNzRESA44Gn3XIPA99v53obY4zZD8kMkmHAJs/7MndaU/NE5H0ReVpEDnSnrQbmikiWiAwAZgIHAv2BXaoabmWdxhhjOkgyg0RiTNMm758DRqjqeGAJTgsDVS0GFgNvAo8DbwHhONfpbFzkIhEpFZHS7du3t+0TGGOMaVUyg6QMpxVRbziw2VtAVXeqao379j5gkmfeTe51kyKcAFkP7AD6iEiguXV6lr9XVfNVNX/gwIHt8oGMMcbsK5lBsgI43L3LKg04C1jkLSAiQzxvTwXWudP9ItLffT0eGA8Uq6oCrwCnu8v8CPjfJH4GY4wxrQi0XqRtVDUsIpcCLwJ+4AFV/VBEbgBKVXURcJmInIpz2uobYL67eBBY5lxbZw9wrue6yFXAEyJyI/AecH+yPoMxxpjWifMlv3vLz8/X0tLSVFfD9FSqUFsBlduhcidU7XBf74CacvAFnB9/oOG1Lwg+vzs96Jke8Ezze8q2YXnxgcS67GiMQ0RWqmp+a+WS1iIxplurrXSCoHJH42Co3A5VOz3v3Wl1NbHXI37Quo6tu1fcQeRvKNdc2ViBFdfy3nKxpnu305blgxaYSWZBYgxAaG8zQRAjGKp2QKgq9noCGZA9ELIHOP8OGgPZ/Z3XWe407/u0LKfFEqmDSNj9CTW8rwu50+rc6fXT6jxl3fnRsq0sHwlDXbiF5T0/MZevg3AtRKr2f1sa6djfaz3x7UcQNRd6sVp5nrKxWomNArYDlk9RK9OCxHRP4RpPEOxo3DrY5/1O59RTLP60hmDIGgADRrkhMcATDJ73adn7/4cs4hys/D3gzzEScVpgcYeWN4g8IRlr+UYh29zy+1PWnV9b1czyLdUrlLp93DSIFrwEAw5L6iZ7wP9c0y2EaxtaBvEEQ82efVYRAUK+AOGcgYQy+xHK6ke4zwRCmX0JZ/YmlN6LcEZvQhk5hNKyCaVlE/YHCWmYUCREOBImVBcirM6/oUiIcO1XhKo3Et7uKRMJNXrddDnv8gA5wRyyg9nRn5w05713ek4wh+y0xtMy/BlIVztl4/MBPufbc3fnbWXGG1qNAjVW6HlakfEun9Gr9bomyILE7Le6SF3DQTES3veg2eSAGvMAG64htHcX4epdhGp2E67ZQ6imnHBtBaFQBeFQFaHQXkKhvYTrqp3lgJAIYRFCAiHc1/4gYV+AUNBPuG+AUL8DCMkQwighVcJECEXqiOA9rVLu/FR9Ac2cpdofghD0BQn4AgT9QQLS5F9fgKAvGC2TEcggx5eDouwN7aWsoozKUCUVoQoqaysJR29SbJ5f/NGQyQpmNQqbRtM8QZQdiB1UAZ8dCtqdz+/8kJ7qmiSd/e9JoXgOyC29jvdAHnNeXQvzWlk+ksTz3KJKEAggBMVHIN1H0NcreiAO+NMI+tMJBjIJBDLICGSS4w80HMQ9B+tYB/am8/cp650WIxBilQ/4AvjF326tA1WlNlJLRW1FQ7iEKhsFzT7T3H93V+9mc8XmaJmqcHwpmeHPiLaGsgJZzbeKPC2mRtPcfzMDmV2vlWQSZkHSgn989g8+3/15ywfbVg7ILc1L5gEZIM2Xtt/fkKMHSAkS1AiBSJhgXYhAuJZgXS2BUDXBcDXB2r0Ea6sIhCoJ1lQQrKkkQISgQkCVYH0gKATTcwmk9yGY2ZdAZh+CWf0JZg4gkNWfYPZAAjmDCWYPIpBzAP6s/khPuFbQAhEh3Z9OemY6/TP7J7SuukgdVeGquIMoOq22gi0VW5wwClVRHionHGm9leQTH9mB7H1Ow+0TRDHKNA2lYE84/dVN9Oy/2FYs/mwxy75aFj3gtumA3Fm+IUciUL2r+TuSyndA5RbPdYidzd9hk9m34UJz74ObvyMpe6BTtocHQyr5fX5y03LJTctNeF21dbXRIKoMVzbfYgo1zKt//XXl19F5VaEqNHYXeY2k+dJitoyau2bUNIjqQyszkIlPbAy/ZLIHEltQF6nDJ77O2VRXherdMYIhxoXn+n+b+0aZ0Tv2HUix3mf16xkXSk3SRDTC3vDe1oMoXNmo9VTfOvKWqY3Utro9QcgOZkevGXlbRnGdxvO0ntL8PWscPXsgsR34ff6O25iq85Rzq3cked43d4thei/IclsGfQ6CYRNbCIb+EOhZfxwmtXziix6wExWqCzV/eq6V03jbqrY1Cqt4WklBX3DfGxlauGbUXOspK5DVsceXJLMgSRZV5+nn/QmG5p5+TstpCIZew2DIhJaDIZjRsZ/VmBQJ+oP08fehT0afhNajqk4raT+CqH7a9qrtbAxtjLaYquuq49pmViCr0d103rBpLYi8r9P96Sk/a2JBsj9qqzzdYXifaWj63v0J7429nkBmw/WEnMEw6MiGQIheX/AERTCzYz+nMT2MiJAVzCIrmJXwukKRUPQUXEWtc+dcvNeTdu7d2WhePDfkBCTQ4s0NF0+4mMHZgxP+XC3WIalr7+qW/AY+X9oQFKHK2OX86Y0vNA/4Toxg8MxPS7xJb4zpnIK+IL3Te9M7vXdC61FVquuqG4VNqzc2uNO+rf6WsvIyKkIVXDDugnb6ZM2zIGlJpM65EN1vZOw7kqLBkGOdwhlj2pWIkBnIJDOQyYDMAamuTossSFoy+z9SXQNjjOn07OZqY4wxCbEgMcYYkxALEmOMMQmxIDHGGJMQCxJjjDEJsSAxxhiTEAsSY4wxCbEgMcYYk5Ae0Y28iGwHvmjj4gOAHe1YnfZi9do/Vq/9Y/XaP921Xger6sDWCvWIIEmEiJTG0x9/R7N67R+r1/6xeu2fnl4vO7VljDEmIRYkxhhjEmJB0rp7U12BZli99o/Va/9YvfZPj66XXSMxxhiTEGuRGGOMSYgFCSAiD4jINhFZ08x8EZE7RWSDiLwvIhM7Sb1miMhuEVnl/vy6g+p1oIi8IiLrRORDEflZjDIdvs/irFeH7zMRyRCR5SKy2q3X9THKpIvIk+7+ekdERnSSes0Xke2e/bUg2fXybNsvIu+JyD9izOvw/RVnvVKyv0Rko4h84G6zNMb85P49qmqP/wGmAxOBNc3MPwl4HhBgKvBOJ6nXDOAfKdhfQ4CJ7utc4BNgTKr3WZz16vB95u6DHPd1EHgHmNqkzCXA/7ivzwKe7CT1mg/8saP/j7nb/jfgsVi/r1TsrzjrlZL9BWwEBrQwP6l/j9YiAVR1KfBNC0W+B/xZHW8DfURkSCeoV0qo6hZVfdd9XQ6sA4Y1Kdbh+yzOenU4dx9UuG+D7k/Ti5PfAx52Xz8NnCCS3PGb46xXSojIcOBkYGEzRTp8f8VZr84qqX+PFiTxGQZs8rwvoxMcoFxHu6cmnheRIzt64+4phaNwvs16pXSftVAvSME+c0+HrAK2ASWq2uz+UtUwsBvo3wnqBTDPPR3ytIgcmOw6uW4HfgFEmpmfkv0VR70gNftLgWIRWSkiF8WYn9S/RwuS+MT6ptMZvrm9i9OFwQTgD8CzHblxEckB/gb8XFX3NJ0dY5EO2Wet1Csl+0xV61Q1DxgOFIjI2CZFUrK/4qjXc8AIVR0PLKGhFZA0InIKsE1VV7ZULMa0pO6vOOvV4fvLdayqTgTmAv8iItObzE/q/rIgiU8Z4P1mMRzYnKK6RKnqnvpTE6q6GAiKyICO2LaIBHEO1o+q6t9jFEnJPmutXqncZ+42dwGvAnOazIruLxEJAL3pwNOazdVLVXeqao379j5gUgdU51jgVBHZCDwBHC8if2lSJhX7q9V6pWh/oaqb3X+3Ac8ABU2KJPXv0YIkPouAH7p3PkwFdqvqllRXSkQOqD8vLCIFOL/PnR2wXQHuB9ap6m3NFOvwfRZPvVKxz0RkoIj0cV9nArOAj5oUWwT8yH19OvCyuldJU1mvJufRT8W57pRUqnqNqg5X1RE4F9JfVtVzmxTr8P0VT71Ssb9EJFtEcutfA7OBpnd6JvXvMdBeK+rKRORxnLt5BohIGXAdzoVHVPV/gMU4dz1sAKqA8ztJvU4HLhaRMLAXOCvZf0yuY4HzgA/c8+sAvwQO8tQtFfssnnqlYp8NAR4WET9OcP1VVf8hIjcApaq6CCcAHxGRDTjfrM9Kcp3irddlInIqEHbrNb8D6hVTJ9hf8dQrFftrMPCM+/0oADymqi+IyE+hY/4e7cl2Y4wxCbFTW8YYYxJiQWKMMSYhFiTGGGMSYkFijDEmIRYkxhhjEmJBYowxJiEWJKZHE5FTReTqdlzfqyKSH2N6vojc6b6eLyJ/bGb5iljTk0VEHhKR02NMHyEi53RkXUzXZQ8kmh7NfYhsUQdspxTYZ5yITmwEcA5Od+nGtMhaJKbbcr9VfyQiC0VkjYg8KiKzROQNEVkvIgXe1oH77fxOEXlTRD6L9U29yfp/Ic5gQqtF5GbPrB+IM2DUJyJS6JadIbEHQjpERN4SkRUi8h+tbO9u96lpROQZEXnAfX2BiNzovj7X3fYqEfmT+9R6fZlP3BbTfU1aRNNjfOabgUJ3PZe3VC9jLEhMd3cYcAcwHjgC51v2NOAKnO5Tmhrizj8F52Aak4jMBb4PTHF7Ev5Pz+yAqhYAP8fp1qYldwD3qOpk4OtWyi4FCt3Xw4Ax7utpwDIRGQ2cidMTbB5QB/yziAwF/h1nQKMinP3gFeszXw0sU9U8Vf19K/UyPZwFienuPlfVD1Q1AnwIvOT2rfUBzumbpp5V1YiqrsXpw6g5s4AHVbUKQFW9Pc/W9zq8splteB0LPO6+fqSVsstwWgljgLXAVreTwKOBN4ETcHqbXeH2NXYCcChOT7Cvqeo3qhoCnmqy3ng/szEx2TUS093VeF5HPO8jxP7/7y3f0oh7QvPjOdSvo66ZbTQVV4d3qvqViPTF6ep9KdAPOAOoUNVyt1fjh1X1mkYVFTmtlVXH+5mNiclaJMa0TTHwYxHJAhCRfm1czxs09Fz7z3GUfwvnlNlSnBbKFe6/AC8Bp4vIoPo6icjBwHLgOBHpK87YHfPi2E45zrj3xrTKgsSYNlDVF3Du9ip1TyNd0cZV/QxnRLsVOIMztWYZzjWYDTijPfZzp+GemroWZ8jV94ESYIiqfgX8FmfY4SU4p8V2t7Kd94GweyOBXWw3LbJu5I3pAUQkR1Ur3BbJM8ADqvpMqutlugdrkRjTM/zGbTmtAT6ng8aqNz2DtUiMaYGIjGPfu6lqVHVKd9qmMYmwIDHGGJMQO7VljDEmIRYkxhhjEmJBYowxJiEWJMYYYxJiQWKMMSYh/x/u1VNhZ386FwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c1ebda6240>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2.best_score_, gsearch2.best_params_))\n",
    "test_means = gsearch2.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2.cv_results_).to_csv('maxdepth_min_child_weights.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'min_child_weght' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "max_depth = [4,5,6]\n",
    "min_child_weight = [1,2]\n",
    "param_depth_weight_further = dict(max_depth=max_depth, min_child_weight=min_child_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59333, std: 0.00314, params: {'max_depth': 4, 'min_child_weight': 1},\n",
       "  mean: -0.59334, std: 0.00353, params: {'max_depth': 4, 'min_child_weight': 2},\n",
       "  mean: -0.58874, std: 0.00413, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.58980, std: 0.00291, params: {'max_depth': 5, 'min_child_weight': 2},\n",
       "  mean: -0.58927, std: 0.00345, params: {'max_depth': 6, 'min_child_weight': 1},\n",
       "  mean: -0.58919, std: 0.00385, params: {'max_depth': 6, 'min_child_weight': 2}],\n",
       " {'max_depth': 5, 'min_child_weight': 1},\n",
       " -0.5887351903261084)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_depth_weight_further, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_2.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_2.grid_scores_, gsearch2_2.best_params_,  gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3、重新调整弱学习器数目"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit1(alg, X_train, y_train, useTrainCV=True, cv_folds=None, early_stopping_rounds=10):\n",
    "    \n",
    "    if useTrainCV:\n",
    "        xgb_param = alg.get_xgb_params()\n",
    "        xgb_param['num_class'] = 3\n",
    "        \n",
    "        xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "        cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "                         metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "        \n",
    "        n_estimators = cvresult.shape[0]\n",
    "        alg.set_params(n_estimators = n_estimators)\n",
    "        \n",
    "        #result = pd.DataFrame(cvresult)   #cv缺省返回结果为DataFrame\n",
    "        #result.to_csv('my_preds.csv', index_label = 'n_estimators')\n",
    "        cvresult.to_csv('nestimators_last.csv', index_label = 'n_estimators')\n",
    "        \n",
    "        # plot\n",
    "        test_means = cvresult['test-mlogloss-mean']\n",
    "        test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "        train_means = cvresult['train-mlogloss-mean']\n",
    "        train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "        x_axis = range(0, n_estimators)\n",
    "        \n",
    "        pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "        pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "        pyplot.legend()\n",
    "        pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "        pyplot.xlabel( 'n_estimators' )\n",
    "        pyplot.ylabel( 'Log Loss' )\n",
    "        pyplot.savefig( 'n_estimators_last.png' )\n",
    "    \n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    train_predprob = alg.predict_proba(X_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "        \n",
    "    #Print model report:\n",
    "    print (\"logloss of train :\" )\n",
    "    print (logloss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of train :\n",
      "0.5220845701932315\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmcHHWd//HXp4+5j8xkJvdJEiCAGDFyqrAKcqgcohEUFVxlXRdvdxd3/anrrivuul4LHqjAKgqLiorIeoAoKlcSucMVck7uazL31f35/fGtmXQmPckkpKdmpt/Px6Me011VXf3p6p5+d32r6lvm7oiIiAAk4i5ARERGD4WCiIgMUCiIiMgAhYKIiAxQKIiIyACFgoiIDFAoiOQws38ys+/EXYdIXBQKY4yZVZnZGjN7W864ajNbZ2Zvzhm32MzuNLNdZtZsZivM7HNmVhdNv9zMMmbWFg2rzOxvC1z7GWbWVMjnOBj56nH3f3f39xTo+daY2ZmFWHYhjNT7NdbWy3inUBhj3L0NuBL4qpk1RqP/A1jm7j8GMLNTgd8DfwaOdvcJwDlAH/DSnMU94O5V7l4FvBn4DzN72ci8EjkYZpaKuwYpEu6uYQwOwE3ALcAZwA5gas60PwH/fYDHXw78adC4h4G35dw/H3gKaCaEzMKcaQujcc3RPOfnTDsPWAG0AhuAjwOVQCeQBdqiYdoQr+s64JfR4x8C5g1jfRwN/BbYCTwLLDmUeoDPADdHj5sDOHAFsB7YBbwPeAXwePTar815nnnA76L3YzvwA2BCNO370XN1Rs/1D8NYx2uAf4yeqxtIRfc3RK/lWeC1edbFycBmIJkz7iLg8ej2icAyoAXYAnxpiHV6BtA0xLRa4HvANmAt8EkgEU1LAv8VrYPVwFXRekwNsaw1wJlDTHsvsDJ6X+/o/8wABnwZ2ArsjtbRcUO933H/v46lIfYCNBziGwd1wKboH++KnPGVQAY44wCPv5ycUIi+6JqBI6P7RwLtwFlAGviH6J+zJLq/Evin6P5ron/Ao6LHbgJelVPnCdHtIb9kcuq4KfoCODH6EvwBcOsBHlNJ+NK+InrMCdF6OfZg6yF/KHwTKANeB3QBPwMmAdOjL6XTo/nnR+urFGgE7gO+krPsvb789reOc+Z/FJgJlANHRa9zWk59eQMTeAE4K+f+j4Cro9sPAO+IblcBJw+xjCHfL0Ig/Byojup4DvjraNr7CF/KM6L1fTeHEArR52p79H6WAv8N3BdNOxtYDkwgBMRCoh9GQ73fGoY3qPlojHL3XYRfmBXA7TmT6gjNgpv7R5jZf0T7FdrN7JM5854cjW8jbCV8H3g+mvZW4Jfu/lt37wW+SPhiOpXwS7QKuMbde9z9d8CdwKXRY3uBY8ysxt13uftfDvLl3e7uD7t7HyEUFh1g/jcAa9z9Rnfvi57vJ4QmscNRz7+6e5e7/4bwJX6Lu2919w3AH4GXAbj7ymh9dbv7NuBLwOn7We7+1nG/r7n7enfvJIR9afRa0u6+xt1fGGLZtxC9H2ZWTfj1fEvO+phvZg3u3ubuDx7MyjCzZFT7J9y91d3XELYM3hHNsgT4qrs3RZ/Taw5m+TneDtzg7n9x927gE8ApZjYneg3VhC1Ec/en3X1Tzut7Me93UVMojFFmdhnhF9rdwBdyJu0iNFNM7R/h7v/gYb/CTwm/pPs96O4TPOxTmAIcC/x7NG0aoVmgfxlZwq/U6dG09dG4fmujaQAXE76E1prZH8zslIN8eZtzbncQAmh/ZgMnRQHXbGbNhC+UKYepni05tzvz3K8CMLNJZnarmW0wsxbgZqBhP8vd3zrutz5n+krgw4Stma3Rc00bYtk/BN5kZqXAm4C/uHv/c/01YSvlGTNbamZv2E+N+TQQthDX5ozLff+n5dY96PbBGLx+2ghNc9OjHyLXEpoat5jZ9WZWE836Yt/voqZQGIPMbBKhPfW9wN8AS8zs1QDu3k5oh3/TwSzT3bcQfl2/MRq1kfBl2/+cRmjG2BBNm2lmuZ+fWdE03H2pu19AaGL5GXBb/9McTE0HYT3whyjg+ocqd//bEa7n89Eyj3f3GuAyQtNGv8HPt791nPcx7v5Dd39l9Dhn7x8EufOtIHyhngu8jRAS/dOed/dLCevjC8CPzaxy+C+T7YRf47Nzxg28/4Tmmxk502YexLJzDV4/lcBE9nzOvubuLyf8mDkS+Pto/FDvtwyDQmFsuhb4mbvfG20y/wPw7ehXIdH9d5vZ1VGAYGYzgLlDLdDMJhJ2Rj4VjboNeL2ZvdbM0sDHCDs77yeETjvwD2aWNrMzCGFyq5mVmNnbzaw2ahJpITR7QPiFPdHMag/Teuh3J3Ckmb0jqidtZq8ws4UjXE81YSdys5lNJ/qSyrEFOCLn/v7W8T7M7Cgze030PncRtlIy+eaN/BD4IPBqwj6F/uVcZmaN0ZZJczR6yOWYWVnuQNgSvQ34XHQ49Gzgo4Qto/7X9SEzm25mEwg7xw8kPeh5UlH9V5jZoug1/zvwkLuvid7fk6L11h6tj8wB3m8Zjrh3amg4uAG4kPALasKg8fcAn8u5fxJwF+Gfvhl4EvgcMDGafjnhn6X/yJuthDbnSTnLuIiww3A38AeiHbfRtGOjcbujeS6KxpcAvyI0Y7UAS4FX5jzuBkITQDNDH330bzn3z+AAO6ej+Y4iHLG0LVr+7wj7Ig6qHvLvaE7lzN9Ezk58whfhJ3PWyfJofT5K+JJvypn3AmBd9FwfH8Y6XsPeO6aPJ+z7aSXsjL8z3zrMmX8W4Qv8l4PG3xy9322EHwEXDvH4M6LXP3iYT9h3dXO0vtcDn2LP0UcpwpbsDsLRRx8hbFnYEM+zJs9z/Fs07X2Eneb9r3dGNP61hCOO2thzpFfVgd5vDQceLFrBIiIFYWbnAt9099kHnFlip+YjETmszKzczM4zs1TUjPZpwkEOMgZoS0HGBDN7FfB/+aZ5OHpKRgkzqyA0hR1N2O/xS+BD7t4Sa2EyLAoFEREZoOYjEREZMOY62WpoaPA5c+bEXYaIyJiyfPny7e7eeKD5xlwozJkzh2XLlsVdhojImGJmaw88l5qPREQkh0JBREQGKBRERGTAmNunICJS7Hp7e2lqaqKrq2ufaWVlZcyYMYN0On1Iy1YoiIiMMU1NTVRXVzNnzhxC57qBu7Njxw6ampqYO3fI/i/3S81HIiJjTFdXFxMnTtwrEADMjIkTJ+bdghguhYKIyBg0OBAONH64iiYUVm3ZzV0PP4W69RARGVrRhMLd3/4E5911Ku0d7XGXIiIyahVNKJywcB4ALTu2HGBOEZHRb6hWjxfbGlI0oZCuCtdPb9u1NeZKRERenLKyMnbs2LFPAPQffVRWVnbIyy6aQ1LLakM/UJ0t22KuRETkxZkxYwZNTU1s27bv91n/eQqHqmhCoWJCCIXu3QoFERnb0un0IZ+HcCBF03xUXTcZgL62HTFXIiIyehVRKEwCINuhUBARGUrRhEKypIx2ykh07oy7FBGRUatoQgGg1WpIdu2KuwwRkVGrqEKhPVlDSe/uuMsQERm1iioUOtO1lPc2x12GiMioVVSh0FNSR1WmJe4yRERGrYKFgpndYGZbzezJIaabmX3NzFaa2eNmdkKhaumXLZ1AtbcW+mlERMasQm4p3AScs5/p5wILouFK4BsFrAUAL6+n1trp7Oou9FOJiIxJBQsFd78P2N/xnxcA3/PgQWCCmU0tVD0AicqJADTvVKd4IiL5xLlPYTqwPud+UzRuH2Z2pZktM7Nl+fr6GK5Uf6d4O9UpnohIPnGGQr7LA+Xt89Xdr3f3xe6+uLGx8ZCfsLQmhEKH+j8SEckrzlBoAmbm3J8BbCzkE1ZMCF1ddKunVBGRvOIMhTuAd0ZHIZ0M7Hb3TYV8wqqo/6Pe1u2FfBoRkTGrYF1nm9ktwBlAg5k1AZ8G0gDu/k3gLuA8YCXQAVxRqFr61dSHnlKzbQoFEZF8ChYK7n7pAaY78HeFev58UuXVdFCGdSgURETyKaozmgF2Wy3pLoWCiEg+RRcKbel6Srt1TQURkXyKLhS6Suqp7lP32SIi+RRdKPSWNVCbVU+pIiL5FF0oZCsbqaOFDvV/JCKyj6ILhWRVI0lzdm5T/0ciIoMVXSika6cAsHtHQU+eFhEZk4ouFComhFDo2FnQk6dFRMakoguF6obQO3fPbjUfiYgMVnShUNsQeufOtCoUREQGK7pQKKmqp5cktOuaCiIigxVdKJBIsNtqSHXqrGYRkcGKLxSA1qS6uhARyacoQ2FTXzUl6hRPRGQfRRkKlROn0UgzofduERHpV5ShkK2aSgPNtHaqqwsRkVxFGQrJCVNJWZbtWzbEXYqIyKhSlKFQVjcDgN1b1sVciYjI6FKUoVDdOBOA9h1NMVciIjK6FGUo1E2ZDUDvLoWCiEiuogyFsglTyWDQujnuUkRERpWiDAWSKZqtjnSHQkFEJFdxhgKwO91Aede2uMsQERlVijYUOksbqelVKIiI5CraUOitmMJE30lfJht3KSIio0bRhoLVTKXe2tje3BJ3KSIio0bRhsJvm8JL37FZJ7CJiPQr2lBY8poTAdi9ZU28hYiIjCIFDQUzO8fMnjWzlWZ2dZ7ps83sHjN73Mx+b2YzCllPrvpp8wDo2rZmpJ5SRGTUK1gomFkSuA44FzgGuNTMjhk02xeB77n78cBngc8Xqp7BqhrnAJBtXj9STykiMuoVckvhRGClu69y9x7gVuCCQfMcA9wT3b43z/TCKamg2WpJtaqnVBGRfoUMhelA7s/wpmhcrseAi6PbFwHVZjZx8ILM7EozW2Zmy7ZtO3znFjSXTKGqa9NhW56IyFhXyFCwPOMGX+rs48DpZvYIcDqwAejb50Hu17v7Yndf3NjYeNgK7CifSl3vZl2BTUQkUshQaAJm5tyfAWzMncHdN7r7m9z9ZcA/R+N2F7CmvWSqpzOVHezu6BmppxQRGdUKGQpLgQVmNtfMSoBLgDtyZzCzBjPrr+ETwA0FrGcfqfrZVFg3mzZvPPDMIiJFoGCh4O59wFXAr4Gngdvc/Skz+6yZnR/NdgbwrJk9B0wGPleoevKpaJwLwK6NL4zk04qIjFqpQi7c3e8C7ho07lM5t38M/LiQNexP3bQQCh06V0FEBCjiM5oBqicfAUBm59qYKxERGR2KOhSsop5Oyki06LKcIiJQ5KGAGTvTU6nsUCiIiECxhwLQXjmDht6NZLM6V0FEpOhDoa92DjPZypaWzrhLERGJXdGHQknjEZRbDxub1sRdiohI7Io+FGqmHQlA84bnYq5ERCR+RR8K9TOOAqBrq05gExEp+lBI1c8mQwLbtTruUkREYlf0oUCqhJ3JRiradK1mERGFAtBaPoO6no3qQltEip5CAeitnc0M38z2NnWhLSLFTaEApBsXMNFaWd2kLrRFpLgpFIDamccAsGPtkzFXIiISL4UCUD/7WAC6Nj0TcyUiIvFSKABWN4deUqR2Ph93KSIisVIoACTT7CyZRnX7mrgrERGJlUIh0l59BNP7mmjr7ou7FBGR2CgU+jUsYLZtZtWW5rgrERGJjUIhUjl9ISWW4T9v/U3cpYiIxEahEJk4+zgAzp/eGnMlIiLxOWAomNk8MyuNbp9hZh80swmFL21kpSYfDYBvfTbmSkRE4jOcLYWfABkzmw98F5gL/LCgVcWhrJZdqUlUt+iwVBEpXsMJhay79wEXAV9x948AUwtbVjxaa49kTmYNO9q64y5FRCQWwwmFXjO7FHgXcGc0Ll24kuJjk49lnm3guY074y5FRCQWwwmFK4BTgM+5+2ozmwvcXNiy4jFhzvGUWIaNq9QHkogUp9SBZnD3FcAHAcysDqh292sKXVgcqme9FIDOpieAs+ItRkQkBsM5+uj3ZlZjZvXAY8CNZval4SzczM4xs2fNbKWZXZ1n+iwzu9fMHjGzx83svIN/CYdRw5H0kaBt/eOxliEiEpfhNB/VunsL8CbgRnd/OXDmgR5kZkngOuBc4BjgUjM7ZtBsnwRuc/eXAZcAXz+Y4g+7VCnN5bOZ7+vo6s3EWoqISByGEwopM5sKLGHPjubhOBFY6e6r3L0HuBW4YNA8DtREt2uB2K9y09twDEfbWlZsaom7FBGRETecUPgs8GvgBXdfamZHAMM5mH86sD7nflM0LtdngMvMrAm4C/jAMJZbUJVzXs4M286zL6yOuxQRkRF3wFBw9x+5+/Hu/rfR/VXufvEwlm35Fjfo/qXATe4+AzgP+L6Z7VOTmV1pZsvMbNm2bduG8dSHrvqIVwDQsnpZQZ9HRGQ0Gs6O5hlm9lMz22pmW8zsJ2Y2YxjLbgJm5tyfwb7NQ38N3Abg7g8AZUDD4AW5+/XuvtjdFzc2Ng7jqQ+dTQ1HIKW3PFbQ5xERGY2G03x0I3AHMI3Q/POLaNyBLAUWmNlcMysh7Ei+Y9A864DXApjZQkIoFHZT4EDKatlVNotpHc/S2tUbaykiIiNtOKHQ6O43untfNNwEHPDnetQ1xlWE/RFPE44yesrMPmtm50ezfQx4r5k9BtwCXO7ug5uYRlzv5OM5LrGax5t2x12KiMiIOuDJa8B2M7uM8KUNYT/AjuEs3N3vIuxAzh33qZzbK4DThlfqyKk5YjFla+/kl8+/wGnz92nNEhEZt4azpfBuwuGom4FNwJsJXV+MW2Wzw87m1hceirkSEZGRNZyjj9a5+/nu3ujuk9z9QsKJbOPXtBPIkKR621/oy2TjrkZEZMQc6pXXPnpYqxhtSiponXA0L8k+yzObdSU2ESkehxoK+c5BGFdSc05mUeIFlq/aGncpIiIj5lBDIfYjhAqtat6pVFg3G57VSWwiUjyGPPrIzFrJ/+VvQHnBKhotZp4IgG14mGz2EhKJcb9xJCIydCi4e/VIFjLq1M6ks2wyx7WvYMWmFo6bXht3RSIiBXeozUfjnxnMeSUnJ57m/pXxnmQtIjJSFAr7Ub7gdBptN6uffTTuUkRERoRCYX/mvBKA5No/092ni+6IyPinUNif+iPoKp/MSYkVPLRqZ9zViIgU3HC6zm41s5ZBw/qoO+0jRqLI2JiRMueUxArufXpz3NWIiBTccLYUvgT8PaHb7BnAx4FvEy6veUPhShsdUq/7FxqshXVPP8wo6MBVRKSghhMK57j7t9y91d1b3P164Dx3/1+grsD1xW/eawA4svVhVm1vj7kYEZHCGk4oZM1siZklomFJzrTx/9O5ego9jcdxevIxLr/h4birEREpqOGEwtuBdwBbo+EdwGVmVk64iM64V3LUWSxOPMfUsr64SxERKajhdJ29yt3f6O4N0fBGd1/p7p3u/qeRKDJ2888kRYa6LX9m/c6OuKsRESmY4Rx9NCM60mirmW0xs5+Y2YyRKG7UmHkSmbI6Xpdcxl1PbIq7GhGRghlO89GNwB3ANMIRSL+IxhWPZJrkUedyZuIRvn7P03FXIyJSMMMJhUZ3v9Hd+6LhJqCxwHWNPgvfSK21c3zfEzy/RRfeEZHxaTihsN3MLjOzZDRcBuwodGGjzry/wtMVnJtcyu2PbIi7GhGRghhOKLwbWAJsBjYBbwauKGRRo1K6HDvybN6QXs6df1lLNjv+j8YVkeIznKOP1rn7+e7e6O6T3P1C4E0jUNvo85K3UJNt5oi25dz3vLrTFpHx51A7xPvoYa1irJh/Jl5Wy5LSB7j5wXVxVyMictgdaigU57UpU6XYMRdwli3j/mfWsaG5M+6KREQOq0MNheJtUD/+EkqyHZxrD/Hmb9wfdzUiIofVkKEwRJfZLWbWSjhnoTjNPhXq5/G3NX+mrbuPtm51fSEi48eQoeDu1e5ek2eodvfUSBY5qpjBCe9kftcTTOpey60Pa9+CiIwfBb3ympmdY2bPmtlKM7s6z/Qvm9mj0fCcmTUXsp7DZtHbAOODqZ/xhV89Q1evLtUpIuNDwULBzJLAdcC5wDHApWZ2TO487v4Rd1/k7ouA/wZuL1Q9h1XVJDjuYl5f9hglmQ6++6fVcVckInJYFHJL4URgZdTLag/hSm0X7Gf+S4FbCljP4XXy+0n1tvHJ6Y/w9XtXsq21O+6KRERetEKGwnRgfc79pmjcPsxsNjAX+N0Q0680s2VmtmzbtlFy0tiMl0NJNW9uvoHunh5e/7U/xl2RiMiLVshQyHcuw1CHsl4C/Njd8zbOu/v17r7Y3Rc3No6ivvgu/DrpTAdfOPoFtrd185w6yhORMa6QodAEzMy5PwPYOMS8lzCWmo76Hf0GaFzIha23kDDnzd+4X30iiciYVshQWAosMLO5ZlZC+OK/Y/BMZnYUUAc8UMBaCiORgFd/nOSOZ/le7Xdo6erjBzpEVUTGsIKFgrv3Ea7h/GvgaeA2d3/KzD5rZufnzHopcKu7j82f2MdeBJNfwiklK2ksy/Lpnz+pS3aKyJhlY+27ePHixb5s2bK4y9jbqt/D9y5g92n/zGl/einHTa/hh+85mUSiOLuIEpHRx8yWu/viA81X0JPXisYRZ0B5HbX3X8O/njmZB1ft5Iwv3ht3VSIiB02hcLi8+9cAXNhyM2cdM5mNzV0sXbMz5qJERA6OQuFwaTwKKidhS7/Nl09PkkwYl17/IGu2t8ddmYjIsCkUDqf33w8VDVT95mPc9YFTqS5L8c4bHmZra1fclYmIDItC4XCqqIdzvwAbljPv1tOZNqGcpl0dvOuGpbR09cZdnYjIASkUDrfjLoajzoOWDfxyyQRuuuJEntnUwimfv4ed7T1xVycisl8KhcPNDM7/b8DhO2fx6lklXP/OxXT0ZDj1mnt0CU8RGdUUCoVQ2QCX/RSyvfDjd3PW0Q3c+t6TSScTXPz1+9VHkoiMWgqFQpn7KpgwG1beDb/9FCcdMZHb/uYUdrR3c+5X/8gyHa4qIqOQQqGQPvgXOPFKeOBaWH4TC6fW8LuPnUEqYbzlmw9w7e+epy+TjbtKEZEBCoVCO/vzMP9M+MWH4fHbmFlfwcP/dCb1lSV88TfP8dbrH2TtDp3LICKjg0Kh0JIpWPJ9KK2B298LT/2U2oo0y//fWXz1kkU8ur6Zv/ri7/nOH1eRUbfbIhIzdYg3Unra4eaLYf3DsOR/YOEbAdi0u5Nzv/JHmjt7qSxNMru+grs+9OqYixWR8UYd4o02JZXw9h9BugL+9x3w7K8AmFpbziOfOouvXfoyunuzrNjUykv/5dcsX7sr5oJFpBhpS2Gkde2GLx0TthwuuA5e9vaBSR09fZz95fto2tWJA7XlaabVlnHXh16FmbrhFpFDpy2F0aqsFj76dPj78/fDH/4TomCuKEnxx398DU/+y9lcfe7RtHf38fTmVo799K+5bdl6OnvyXsJaROSw0ZZCXPp64I4PwOO3QuUk+NBjUFKx1yydPRnO++p9bG7pprM3BEJjVQnXvu0ETpxbr60HERm24W4pKBTi5A5/+AL8/hqYfFzYAT1xXp7ZnIdW7+SqH/6F7W2h/6TSVIKGqlJuvfJkZtZX7PMYEZFcCoWx5Pm74fb3QDYDZ/87LHo7JPK37HX09PGrJzfzmTueoqWrD4DqshR1FSVMKE/z86tO0xaEiOxDoTDWNK+D26+EdQ+Ecxre/SuYfOx+H3LhdX9ixaZWkmYDzUulqQQTKtJRQLySsnRyJKoXkVFOoTAWZbPw6A/gzg9Dtg9O/SCc/o9QWnXAh67f2cE7vvsQzR29tHT10n8eXE1ZipryNLVlYSsildSxBSLFSKEwlrXvgLs/DY98H5KlcPG3YeH5oVvuYejqzXDBtX9i1fZ20skEHdFRSwmDytIUXb0ZZtdXcMPlJzKzvlzNTSJFQKEwHqx7EL5/EfR2wIwT4czPwJzTDnoxO9q6eeu3HqC1u4+27j7au/cc2ppOGu4wtbaMytIUFSVJbn//wT+HiIxuCoXxItMHj94Md/09ZHqgvA7edSdMOe6QF9mbyXLRdX+mLQqJHe095H4MDKitSFORTrKzo4f5jVXc9r5TqChJvfjXIyKxUCiMNz0d8PC34Hf/Go5SOv6tYX9DnkNYD0VzRw+XXP8gq7e3k8k6qaTR3Zsl99NhBgkzGqtKKEsn2dLSxdFTarj9/aeqCUpklFMojFedu+BPX4H7vwaehfKJ8LZbYeaJh/2pejNZLv76/XT2ZujsybC5pYts9HnJ7dA1YVCWTtLTl2VyTRnl6QQbdnexYFIVP3jPSVSVphQaIjFTKIx3bVvhoW/Bn78SjlSaeTKc+gE46rwhz3E4XNydi75+P89ubiHrMKEiTVdvhpbOPvJ9mswAh4rSJOlkgvbuPgyYOqGcdDJBOmFc9/YTaKgupVoBIlIQCoVi0d0Gj9wMv/0UZLohVQav+X+w6G1QUT/i5XT1ZnjLN+9n5dY2HGisKqU342xv66aiJElf1unsyeQNj34GVJQkSadyAqS2nHTSwhZIYxXff89J1JQpQESGa1SEgpmdA3wVSALfcfdr8syzBPgM4MBj7v62/S1ToTCETB+s+Bk8fD2sfwgweOklsPjdMOMVwz6cdaQs+eb99GWd3kyWlVvbmDahnN5Mlk27u3CHytIkvZnhB0gyYXT0ZKirKCGZMHZ29GDA9AnlpBLG+uZO5jdWkjRj5bY2jplaw23vO3WEXq1I/GIPBTNLAs8BZwFNwFLgUndfkTPPAuA24DXuvsvMJrn71v0tV6EwDJufhOU3wrIbwTOhX6XFV8BLlkBZTdzVHbQDB0iKTNbp6OkjmTAyWWc4F7FLJoxkwujLZDGgqixNMmG0dvVSX1lC0ozt7SFcZtaVk0wY63aFcElE4WKggJExYTSEwinAZ9z97Oj+JwDc/fM58/wH8Jy7f2e4y1UoHITuNnjiR/Cbfw7Xb7AELHgdHHVu2PdQNSnuCgumN5Plrd96gEzWyWSdF7a1MaOugr6ss7G5E3enrrKETNZp7ujFgbJUgow73b1ZEmZkDuJ/I5kwslmnNJ0gmTC6ekPQ1JanSZjR3NlDY1UpiYSxrbUbCFsxZrChuRMDZk+sJGGwZkcH86LgMYPvvusVlKQSlKaSpJOmJjM5JKMhFN4MnOPu74nuvwM4yd2vypnnZ4RPLnPpAAAQ+UlEQVStidMITUyfcfdf5VnWlcCVALNmzXr52rVrC1LzuOUOG5aHgFh2Y9j3ADDnVXDsheFs6XEcEIdqyTfvZ8WmFhw4oqGKTDbL6u3tzKirIJN1NjSHiyE1VJaQcWdHew81ZWkyWaetKwRNOpkg605v5vD9nxmhNTDr4XY6mcAsBGF5OknCjI7eTE4oQXNnLwCTqktJmLG1tZuptWWYGZt2dwIwq64CM1i3s4O5DZV84eKXkk4ZJckEJanEwN90MgRf/18ZG0ZDKLwFOHtQKJzo7h/ImedOoBdYAswA/ggc5+7NQy1XWwovkjtseQpuuQTat0Nf+EJg7qvh2ItCQFQ2xFvjOOXudPdl6e7LcsWND5ONmrlWbmsDYHZ9BVl31u3sYHpdBe5O067w/kyuKSXrsLWli4aqcHtHewj32vI07rC7s5fK0hRZd9q7Qw+6hQilwQaHVCppGEZvNktpKoFhdPeFs+grSlKYQUdPJhyqDLRFtdaUpzHC65hQEW73h1l9ZQkG7GgPW1xmsC3qRn5ydbi/pSWsjxB2sKm5i+l15RjQ1BwFX30FRgi+2RMrMYO1O/pD8HiSCSMVNSumEgmSyXA/YdH4ZLjdH4W5G217xu49vn95cW/hjYZQGE7z0TeBB939puj+PcDV7r50qOUqFA4jd9i6Ap76GTz1U9jxfBg/+5Ww4EyYf1boqVXNFeNGNut09WXo6cvS05flyu8t4/noSLEjGirJurN6ezuz6itwh3W7OgCYVltO1p1Nu7uYVF2KA1tbu8GdiVWleE5ITagowT00y9WUp3F3WrvC4cqVJUkc6Ojuo6wkiXs4Yg2gJJUAh+5MlnQy3O7NZAFIJAx3HwiesXXM5B65AZpMhBjJRDvASlLhUPL+19zfw3FXb4bykiSGMbW2jN9+9PRDe+5REAopQtPQa4ENhB3Nb3P3p3LmOYew8/ldZtYAPAIscvcdQy1XoVAg/VsQK34GD3wdetvD+GRJOIpp/lkw91Whmw2RUaC/eW/hlBoceGZzaOo7clI1jvP8ljbmT6rCc7bG5jZU4g6rd7Qzp74CB9buCJ/1GXXhYlXrd3UwfUI57oT9T4StD3fY3NIFhC039xCMk6pLAdgS7SuanHN/UnUpOGxtDY+bWBWmbW/rpr6yBHfY1RG2eHK3+CBcJ8UdWrv7qCpN4e5MqS3jNx8Zo6EQFXEe8BXC/oIb3P1zZvZZYJm732Fhe+q/gHOADPA5d791f8tUKIyQlk2w8u7QW2tncziKCaCkEl5+RdgfMfuUcK1pERn1RkUoFIJCIQaZXmhaCqv/CA9cB90tDGzATzke5rwSZp8Ks06Fyomxlioi+SkUpHB6O2H9w7D2fnjwG9DTGvphAmg8GmafFkJi9mlQMzXeWkUEUCjISOrrho2PwNo/h6B44d49zU2pMnjJm0NAzDoF6uZox7VIDBQKEp9MH2x+PATE2vvh+V+HTvsAkulwDerTPgzTXw5TXzqsy42KyIujUJDRI5uFbU+HgGhaGo5w6uveMz1dAQvfCFMXwbRFMOUlUFodX70i45BCQUa3tm2hyWnD8nDxoJ72cGW5fqkymH9mCIjJx4XzJdT0JHLIFAoy9rRuhk2PhaanzU/Cc7/ec8Y1QEl1CIcpx4WgmPISmLQwHCYrIvs13FDQRXdl9KieEoYjz94zrqcdtj4Nm5+ALU/C47eFJqj+HdkAE+dHIXEcTH5JCI7aGdqqEDkE2lKQsccdmteGrYktT4bAWHk39HXtmSeRClejm3wsNCwIw8QFUDNNYSFFSVsKMn6Zhf0LdXNg4Rv2jO9qCX059W9VPPETWHf/nnMoIHQfni6HI8+FhiOh8cjwt34epMtG+pWIjDoKBRk/ympg1slhAHjjV8NWResm2P586PBv+8rw9+lf7OlCvF+qNHQGOHF+NMwLf2tnQCI58q9HJAYKBRnfzEKTUc00OGJQR2I9HbBjJWx/LgqNlfDcr+CFe/ZdTroC5r0mhETDgj3BUTFRzVEyrigUpHiVVMDU48OQyx3atoSQGBhegJX3wDN37j1vIhnOrxi8dTFxvo6KkjFJoSAymNmeI6HmvHLvaZk+2L0uaobKGZ762b7NUckSmHlSTmBEw4RZkCoZudcjchAUCiIHI5mC+iPCwOv2ntbTATtXRUHxPDx0PWx8FNY9sKebjwEW9oEccyHUz4W6uXv+ltWM1KsR2YcOSRUZCR0792xV7FwdrpXd1xVOzssXGCVVcNQ5ISTq5oStiwmzoGZ6CCaRg6QzmkXGiq4W2LU6hMWu1aE78r4u6O3at0kKIFkajpRKlcHiK/YExkBopEf+Nciop1AQGQ/6eqClCZrX7T08+3/Q3UreqxX3h8bRr987MBQaRU0nr4mMB6mSnH0YefT1QMuGPKFxFzx2K/sNjaPO3TswameGczJSpQV9STK6KRRExrJUSdhBXT83//Tc0Ni9HnatDX+f+WX+I6YgHDU17QSYMHPvwJgwO4SGzvwe1xQKIuPZgUIj0wstG/eERu7WxtN37H3di37JknBxpJrp0TAtXHa1/3b1VDVRjWEKBZFilkxD3eww5JPpC92EDA6M5nX7dm2eq2pyFBY5QTEQINGQLi/c65JDplAQkaElU1Ez0kyYfeq+092ha3cIjpYNYaujZeOe2ztXwXP/B9nMvo9NpKBx4d5BMTBEAaIr8I04hYKIHDozKJ8QhkkLh56vpx1acoMjJ0DW3BdO/Mu3U9ySoa+p/rCoHhQaNdOgvE79Tx1GCgURKbySSmiYH4ah9HVHWxyDtjb6/675096XbO1niXCS30BYDG6qmg4VDZBIFO71jSMKBREZHVKle66TMZRMX+isMF9orLw7BAew71aHhSawmunR/o3crY3ob9VknS2OQkFExpJkCmqnh4FX5J8nm4WO7fn3cbRsDOdw5F6lL9dA89TUvUOjemoIjapGKK0Z181VCgURGV8SCaiaFIZpL8s/jzt07sofGi0bYNtz8Mxde18LfICF8zaqGkNQVEZ/qybtfbtqUujDaowFiEJBRIqPGVTUh2HKcUPP19Wy58iqtm3QvjU0X7VtC3+b10HTUmjfTt4d5anyKCAmQ/VkqJqS83dKNH7KqNrnUdBQMLNzgK8CSeA77n7NoOmXA/8JbIhGXevu3ylkTSIiw1ZWE4bGo/Y/X6YPOnZEoRENA7e3hGHbc7D6vnAI7z5sz07y6v5hcrTV0b/lMQUqGwp+adiChYKZJYHrgLOAJmCpmd3h7isGzfq/7n5VoeoQESm4ZCp8iVdPPvC8vZ0hJFq3QNvm8Ld1054jr7Y9E7ohydd0VXcEfOiRw19/jkJuKZwIrHT3VQBmditwATA4FEREike6/MBHWcGe8Mjd2piV5wTCw6yQoTAdWJ9zvwk4Kc98F5vZq4HngI+4+/rBM5jZlcCVALNmzSpAqSIio8xww+MwK+SejXy73AfvifkFMMfdjwfuBv4n34Lc/Xp3X+zuixsbGw9zmSIi0q+QodAEzMy5PwPYmDuDu+9w9/5uGL8NvLyA9YiIyAEUMhSWAgvMbK6ZlQCXAHfkzmBmU3Pung88XcB6RETkAAq2T8Hd+8zsKuDXhENSb3D3p8zss8Ayd78D+KCZnQ/0ATuBywtVj4iIHJiu0SwiUgSGe43m0XEKnYiIjAoKBRERGaBQEBGRAWNun4KZbQPWHuLDG4Dth7Gc8ULrZV9aJ/lpvexrrKyT2e5+wBO9xlwovBhmtmw4O1qKjdbLvrRO8tN62dd4WydqPhIRkQEKBRERGVBsoXB93AWMUlov+9I6yU/rZV/jap0U1T4FERHZv2LbUhARkf1QKIiIyICiCQUzO8fMnjWzlWZ2ddz1xMXM1pjZE2b2qJkti8bVm9lvzez56G9d3HUWmpndYGZbzezJnHF514MFX4s+O4+b2QnxVV44Q6yTz5jZhujz8qiZnZcz7RPROnnWzM6Op+rCM7OZZnavmT1tZk+Z2Yei8ePy81IUoZBzvehzgWOAS83smHiritVfufuinGOrrwbucfcFwD3R/fHuJuCcQeOGWg/nAgui4UrgGyNU40i7iX3XCcCXo8/LIne/CyD6/7kEODZ6zNej/7PxqA/4mLsvBE4G/i56/ePy81IUoUDO9aLdvQfov160BBew56p3/wNcGGMtI8Ld7yN0155rqPVwAfA9Dx4EJgy6Fsi4MMQ6GcoFwK3u3u3uq4GVhP+zccfdN7n7X6LbrYTrvkxnnH5eiiUU8l0venpMtcTNgd+Y2fLo2tcAk919E4R/AGBSbNXFa6j1UOyfn6uiZpAbcpoWi3KdmNkc4GXAQ4zTz0uxhMJwrhddLE5z9xMIm7h/Z2avjrugMaCYPz/fAOYBi4BNwH9F44tunZhZFfAT4MPu3rK/WfOMGzPrplhC4YDXiy4W7r4x+rsV+Clhk39L/+Zt9HdrfBXGaqj1ULSfH3ff4u4Zd88SrqPe30RUVOvEzNKEQPiBu98ejR6Xn5diCYUDXi+6GJhZpZlV998GXgc8SVgX74pmexfw83gqjN1Q6+EO4J3RUSUnA7v7mw3Gu0Ft4RcRPi8Q1sklZlZqZnMJO1UfHun6RoKZGfBd4Gl3/1LOpHH5eSnYNZpHk6GuFx1zWXGYDPw0fMZJAT9091+Z2VLgNjP7a2Ad8JYYaxwRZnYLcAbQYGZNwKeBa8i/Hu4CziPsTO0ArhjxgkfAEOvkDDNbRGj+WAP8DUB0vfXbgBWEo3P+zt0zcdQ9Ak4D3gE8YWaPRuP+iXH6eVE3FyIiMqBYmo9ERGQYFAoiIjJAoSAiIgMUCiIiMkChICIiAxQKIiIyQKEgMgxmtmhQt9HnH64u2M3sw2ZWcTiWJfJi6TwFkWEws8uBxe5+VQGWvSZa9vaDeExyHJ8sJjHSloKMK2Y2J7oYyrejC6L8xszKh5h3npn9Kuox9o9mdnQ0/i1m9qSZPWZm90Vdo3wWeGt0oZm3mtnlZnZtNP9NZvaN6EIsq8zs9KhH0afN7Kac5/uGmS2L6vqXaNwHgWnAvWZ2bzTuUgsXQnrSzL6Q8/g2M/usmT0EnGJm15jZiqgH0y8WZo1K0XF3DRrGzQDMIXS7sCi6fxtw2RDz3gMsiG6fBPwuuv0EMD26PSH6ezlwbc5jB+4TLk5zK6F3zAuAFuAlhB9dy3NqqY/+JoHfA8dH99cADdHtaYQuExoJXZH8DrgwmubAkv5lAc+yZ2t/QtzrXsP4GLSlIOPRanfv76NmOSEo9hJ1g3wq8KOoP5tvAf2dv/0ZuMnM3kv4Ah+OX7i7EwJli7s/4aFn0adynn+Jmf0FeIRwxbJ8V/97BfB7d9/m7n3AD4D+7s0zhJ46IQRPF/AdM3sToY8dkRetKDrEk6LTnXM7A+RrPkoAze6+aPAEd3+fmZ0EvB54NOoQbrjPmR30/FkgFfUk+nHgFe6+K2pWKsuznHx98ffr8mg/godOHk8EXkvo9fcq4DXDqFNkv7SlIEXJw0VSVpvZW2DgYusvjW7Pc/eH3P1TwHZC3/itQPWLeMoaoB3YbWaTCRc56pe77IeA082sIbrm8aXAHwYvLNrSqfVwzeQPEy6CI/KiaUtBitnbgW+Y2SeBNGG/wGPAf5rZAsKv9nuiceuAq6Omps8f7BO5+2Nm9gihOWkVoYmq3/XA/5nZJnf/KzP7BHBv9Px3uXu+61tUAz83s7Jovo8cbE0i+eiQVBERGaDmIxERGaDmIxn3zOw6wtWzcn3V3W+Mox6R0UzNRyIiMkDNRyIiMkChICIiAxQKIiIyQKEgIiID/j/d1lZT9kBvXQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c1ebc4dd68>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#调整max_depth和min_child_weight之后再次调整n_estimators(6,4)\n",
    "xgb3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit1(xgb3, X_train, y_train, cv_folds = kfold)\n",
    "#from sklearn.model_selection import cross_val_score\n",
    "#results = cross_val_score(xgb2_3, X_train, y_train, metrics='mlogloss', cv=kfold)\n",
    "#print results\n",
    "#print(\"CV logloss: %.2f%% (%.2f%%)\" % (results.mean()*100, results.std()*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_estimators_last is 220\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "cvresult1 = pd.DataFrame.from_csv('nestimators_last.csv')\n",
    "print('n_estimators_last is',cvresult1.shape[0]) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.4、行列重采样参数调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_test4 = dict(subsample=subsample, colsample_bytree=colsample_bytree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59017, std: 0.00373, params: {'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  mean: -0.58785, std: 0.00399, params: {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  mean: -0.58639, std: 0.00301, params: {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  mean: -0.58574, std: 0.00326, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: -0.58558, std: 0.00323, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: -0.58481, std: 0.00278, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: -0.58911, std: 0.00396, params: {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  mean: -0.58772, std: 0.00400, params: {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  mean: -0.58590, std: 0.00399, params: {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  mean: -0.58557, std: 0.00404, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: -0.58492, std: 0.00388, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: -0.58473, std: 0.00406, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: -0.58874, std: 0.00413, params: {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  mean: -0.58771, std: 0.00465, params: {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  mean: -0.58699, std: 0.00408, params: {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  mean: -0.58574, std: 0.00365, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: -0.58515, std: 0.00361, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: -0.58443, std: 0.00338, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.58787, std: 0.00444, params: {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  mean: -0.58723, std: 0.00390, params: {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  mean: -0.58552, std: 0.00344, params: {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  mean: -0.58487, std: 0.00399, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: -0.58401, std: 0.00377, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: -0.58395, std: 0.00387, params: {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " {'colsample_bytree': 0.9, 'subsample': 0.8},\n",
       " -0.5839484573688255)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb4 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch4 = GridSearchCV(xgb4, param_grid = param_test4, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch4.fit(X_train , y_train)\n",
    "\n",
    "gsearch4.grid_scores_, gsearch4.best_params_,     gsearch4.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.583948 using {'colsample_bytree': 0.9, 'subsample': 0.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xd4VEX3wPHvpEBICAkQSui9pxA6CEgPEnpRUIoKCIqA/PQFRaQor/iKDVBAUEFpCkiX3pv0EEowhJqQEEJPIXXP748NMUBCNmQ3jfk8zz5md+/MnbuGPZl775yjRARN0zRNe1ZW2T0ATdM0LXfTgUTTNE3LFB1INE3TtEzRgUTTNE3LFB1INE3TtEzRgUTTNE3LFB1INE3TtEzRgUTTNE3LFB1INE3TtEyxye4BZAUXFxepUKFCdg9D0zQtVzl27NhNESmW3nbPRSCpUKECR48eze5haJqm5SpKqSumbKdPbWmapmmZogOJpmmalik6kGiapmmZ8lxcI9G0nCA+Pp7g4GBiYmKyeyia9gg7OzvKlCmDra3tM7XXgUTTskhwcDCOjo5UqFABpVR2D0fTABARbt26RXBwMBUrVnymPvSpLU3LIjExMRQtWlQHES1HUUpRtGjRTM2UdSDRtCykg4iWE2X291IHkqf4/chVdp67kd3D0DRNy9F0IElDfKKBXw9eYeTSE5wPi8ju4WiapuVYOpCkwdbaih8H1Ce/rTWDfz3Knai47B6SpmXK3bt3+eGHH56p7bfffkt0dLTZxrJgwQJGjBhhtv5MtWvXLnx8fDLcLqPjvXz5MkuWLMnwfkx1+/Zt2rVrR9WqVWnXrh137txJdburV6/Svn17atasSa1atbh8+bJFxmPRQKKU8lZK/aOUClRKjUvl/UFKqXCllG/SY3CK975QSp1Oeryc4vWKSqlDSqnzSqnflVL5LDX+0s4FmNu/HqF3Y3hnyXHiEw2W2pWmWVxOCiR53dMCSUJCQqb7nzZtGm3atOH8+fO0adOGadOmpbrdgAED+OCDD/D39+fw4cMUL1480/tOjcVu/1VKWQPfA+2AYOCIUmqtiJx9bNPfRWTEY207AV6AJ5Af2K2U2igi94EvgG9EZJlSag7wJjDbUsdRr3xh/tvDjfeXn2TKurN82q2OpXalPUcmrzvD2ZD7Zu2zVqlCTOxcO833x40bx4ULF/D09KRdu3YUL16cP/74g9jYWLp3787kyZOJioqiT58+BAcHk5iYyIQJEwgLCyMkJIRWrVrh4uLCzp07U+1/06ZNfPTRRyQmJuLi4sL27du5ffs2b7zxBhcvXsTe3p4ff/wRd3f3R9otX76cyZMnY21tjZOTE3v27OHy5cv079+fqKgoAGbNmkXTpk3ZtWsXEydOpESJEvj6+tKjRw/c3Nz47rvvePDgAatXr6Zy5coMGjQIOzs7zpw5Q1hYGF9//fUTM5GoqCjeffddTp06RUJCApMmTaJr165pfn5BQUF4e3tz6dIl+vXrx8SJE5kwYQIuLi6MGjUKgPHjx1OiRAmWLFmCv78/np6eDBw4kMKFC7NhwwZiYmKIiopix44dfPnll098/gCLFi1ixowZxMXF0ahRI3744Qesra0fGcuaNWvYtWsXAAMHDuTFF1/kiy++eGSbs2fPkpCQQLt27QAoWLBgmseWWZZcR9IQCBSRiwBKqWVAV+DxQJKaWsBuEUkAEpRSJwFvpdRyoDXQL2m7hcAkLBhIAHrVK0NAWAQ/7rlItZKO9G9c3pK70zSLmDZtGqdPn8bX15ctW7awYsUKDh8+jIjQpUsX9uzZQ3h4OKVKlWLDhg0A3Lt3DycnJ77++mt27tyJi4tLqn2Hh4czZMgQ9uzZQ8WKFbl9+zYAEydOpG7duqxevZodO3YwYMAAfH19H2k7ZcoUNm/eTOnSpbl79y4AxYsXZ+vWrdjZ2XH+/Hn69u2bnHj15MmT+Pv7U6RIESpVqsTgwYM5fPgw3333HTNnzuTbb78FjLOC3bt3c+HCBVq1akVgYOAj+506dSqtW7fm559/5u7duzRs2JC2bdvi4OCQ6jEePnyY06dPY29vT4MGDejUqRNvvvkmPXr0YNSoURgMBpYtW8bhw4dxd3dn+vTprF+/HjCeGjt48CB+fn4UKVKELVu2cP78+Sc+/2LFivH777+zf/9+bG1tefvtt1m8eDEDBgxg8ODBDBs2jPr16xMWFoarqysArq6u3Ljx5E1BAQEBODs706NHDy5dukTbtm2ZNm3aE0HJHCwZSEoDQSmeBwONUtmup1KqBRAAvCciQcBJYKJS6mvAHmiFMQAVBe4mBZiHfZa20PgfMda7BufDIpi09gyViznQtHLq/6A0zRRPmzlkhS1btrBlyxbq1q0LQGRkJOfPn6d58+a8//77jB07Fh8fH5o3b25Sf3///TctWrRIXtBWpEgRAPbt28fKlSsBaN26Nbdu3eLevXuPtG3WrBmDBg2iT58+9OjRAzBmARgxYgS+vr5YW1sTEBCQvH2DBg2Sv0QrV65M+/btAXBzc3tkttSnTx+srKyoWrUqlSpV4ty5c098BmvXrmX69OmAcZ3P1atXqVmzZqrH2K5dO4oWLQpAjx492LdvH6NHj6Zo0aKcOHGCsLAw6tatm7xNau0ffi5pff5+fn4cO3aMBg0aAPDgwYPk01Hz589Ptd+0JCQksHfvXk6cOEG5cuV4+eWXWbBgAW+++WaG+jGFJQNJajcmy2PP1wFLRSRWKTUM4wyjtYhsUUo1AA4A4cBBIMHEPo07V2ooMBSgXLlyz3YEKVhbKWb0rUv3Hw7w9uLjrHmnGeWLpv6Xi6bldCLChx9+yFtvvfXEe8eOHeOvv/7iww8/pH379nzyyScm9ZfaWgSRJ/95Pr7dnDlzOHToEBs2bMDT0xNfX19mzpxJiRIlOHnyJAaDATs7u+Tt8+fPn/yzlZVV8nMrK6tHrj88vp/Hn4sIK1eupHr16uke39P6Gzx4MAsWLOD69eu88cYbabZPOdNJ6/OfOXMmAwcO5PPPP3/qWEqUKEFoaCiurq6Ehoameu2jTJky1K1bl0qVKgHQrVs3/v77b4sEEktebA8GyqZ4XgYISbmBiNwSkdikp/OAeinemyoiniLSDmMAOQ/cBJyVUjZp9Zmi/Y8iUl9E6hcrlm5dFpM42tkyf0B9AN5ceJSImHiz9KtpWcHR0ZGICOOt7B06dODnn38mMjISgGvXrnHjxg1CQkKwt7fntdde4/333+f48eNPtE1NkyZN2L17N5cuXQJIPrXVokULFi9eDBjvmHJxcaFQoUKPtL1w4QKNGjViypQpuLi4EBQUxL1793B1dcXKyorffvuNxMTEDB/v8uXLMRgMXLhwgYsXLz4RMDp06MDMmTOTg92JEyee2t/WrVu5fft28rWYZs2aAdC9e3c2bdrEkSNH6NChA5D+55XW59+mTRtWrFiRfKrq9u3bXLnyZEmQLl26sHDhQgAWLlyY6rWdBg0acOfOHcLDwwHYsWMHtWrVeuoxPitLBpIjQNWku6zyAa8Aa1NuoJRyTfG0C+Cf9Lq1Uqpo0s/ugDuwRYz/x3cCvZLaDATWWPAYnlDBxYEf+nlx6WYUo5b5kmhIdUKkaTlO0aJFadasGXXq1GHr1q3069ePJk2a4ObmRq9evYiIiODUqVM0bNgQT09Ppk6dyscffwzA0KFD6dixI61atUq172LFivHjjz/So0cPPDw8ePll442WkyZN4ujRo7i7uzNu3LjkL7+UPvjgA9zc3KhTpw4tWrTAw8ODt99+m4ULF9K4cWMCAgLSvG7xNNWrV6dly5Z07NiROXPmPDKrAZgwYQLx8fG4u7tTp04dJkyY8NT+XnjhBfr374+npyc9e/akfn3jH5X58uWjVatW9OnTJ/n6g7u7OzY2Nnh4ePDNN9880Vf79u1T/fxr1arFZ599Rvv27XF3d6ddu3aEhoYCxpnPw+tE48aNY+vWrVStWpWtW7cybpzxptijR48yeLDx5ldra2umT59OmzZtcHNzQ0QYMmRIhj9Hk4iIxR7ASxivfVwAxie9NgXokvTz58AZjNdEdgI1kl63w3hN5CzwN+CZos9KwGEgEFgO5E9vHPXq1RNz+/XgZSk/dr3896+zZu9by5vOntW/K1ll4MCBsnz58izZV2Jionh4eEhAQECW7M9SUvv9BI6KCd/1Fs3+KyJ/AX899tonKX7+EPgwlXYxGO/cSq3PixjvCMtW/RuX55/r95m7+yLVijvSs16Z7B6SpmlZ7OzZs/j4+NC9e3eqVq2a3cPJNjqNfCZM7FybCzei+PDPU1Qs5oBXucLZPSRNs7hGjRoRGxv7yGu//fYbbm5u2TSiJy1YsOCZ2m3evJmxY8c+8lrFihVZtWpVqtvXqlWLixcvPtO+8hIlqdxVkdfUr19fHp5bNLc7UXF0/X4/0XGJrB3RjFLOBSyyHy338/f3T/PWUk3Lbqn9fiqljolI/fTa6lxbmVTYIR8/DaxPTHwiQ349SnRc5tMfaJqm5SY6kJhB1RKOzOjrydnQ+3yw3C/Ve+c1TdPyKh1IzKR1jRKM867BhlOhzNgemH4DTdO0PEJfbDejoS0q8U9YBN9sC6BaiYJ0dHNNv5GmaVoup2ckZqSU4r/d3ahbzpkxf5zk9LV76TfStCySk9LI63okmWNKPZKdO3fi6emZ/LCzs2P16tUWGY8OJGZmZ2vN3P71cLa3ZeivRwmPiE2/kaZlgZwUSPK6nFCPpFWrVvj6+uLr68uOHTuwt7dPTnBpbvrUlgUUd7Rj3oD69JpzgLd+O8rSoY3Jb2P+1M1aLrZxHFw/Zd4+S7pBx9QLHIGuR/K81SNJacWKFXTs2BF7e/s0t8kMHUgspE5pJ77q7ck7S47z0Z+nmd7bPdXsqJqWVXQ9kuerHklKy5YtY8yYMU/dJjN0ILGgTu6uBIRV5bvt56lR0pEhLSpl95C0nOIpM4esoOuR5P16JA+FhoZy6tSp5MzElqADiYWNalOV8zci+HyjP1WKF6RVDcvUTNa0jBBdjyTP1yN56I8//qB79+7Y2to+tc/M0BfbLczKSjG9twc1ShZi5NITBN5Iu0aBplmSrkfyfNUjeWjp0qX07dv3qceWWXpGkgXs89kwb2B9us7ax5sLj7L67WYUdsiX3cPSnjMp65F07NgxuR4GQMGCBVm0aBGBgYF88MEHWFlZYWtry+zZs4F/65G4urqmerE9ZT0Sg8GQfI1j0qRJvP7667i7u2Nvb59mPZLz588jIrRp0ya5HknPnj1Zvnw5rVq1ylQ9krCwsDTrkYwePRp3d3dEhAoVKiRf00jNw3okgYGB9OvX74l6JM7OzqnWIxk0aBCFCz+a0LV9+/b4+/s/8fmnrEdiMBiwtbXl+++/p3z58o9cIxk3bhx9+vThp59+oly5cixfvhww1iOZM2dO8mmwy5cvExQURMuWLTP8+WWETtqYhY5duU3fHw9Rv0JhFr7REFtrPSF8nuikjVln0KBB+Pj40KtXr/Q3ziSDwYCXlxfLly/P1ankddLGXKJe+SL8t4cbBy7c4tP1Z7N7OJqmZdLZs2epUqUKbdq0ydVBJLP0qa0s1qteGQLCIvhxz0WqlXDktcbls3tImpYhuh7Jv3Q9EiMdSLLBWO8anA+LYNLaM1Qq5kDTyqnfm69pOdGhQ4eyewgW06FDB4veJptX6VNb2cDaSvFd37pUcHHg7cXHuXIrKruHpGma9sx0IMkmhexsmT+gPiIweOFRImLis3tImqZpz0QHkmxUwcWB2a96cfFmFKOW+ZJoyPt30GmalvfoQPI0iZafJTSt4sKkzrXYce4G/9t8Lv0GmvaMclL2X51GPnNMSSMP8J///IfatWtTs2ZNRo4cabHqrTqQpEUElr0K68dA/AOL7qp/kwq81rgcc3df5M/jwRbdl/b8ykmBJK/LCWnkDxw4wP79+/Hz8+P06dMcOXKE3bt3Z3rfqbFoIFFKeSul/lFKBSqlxqXy/iClVLhSyjfpMTjFe/9TSp1RSvkrpWaopMQ2SqmXlVJ+Se/9z2KDNyRCsWpw9Cf4sRWEWXbdx8TOtWlSqSjjVp7i+NXU/7rQtMxImUb+gw8+4Msvv6RBgwa4u7szceJEwJhavVOnTnh4eFCnTh1+//13ZsyYkZxGvlWrVmn2v2nTJry8vPDw8KBNmzaA8S/nbt264e7uTuPGjfHz83ui3fLly6lTpw4eHh60aNECMH4RN2/eHC8vL7y8vDhw4ABgnFG0bNmSPn36UK1aNcaNG8fixYtp2LAhbm5uXLhwATAuSBw2bBjNmzenWrVqqa5Yj4qK4o033qBBgwbUrVuXNWvWPPXze5hGvnr16skp3ydMmMB3332XvM348eOZMWMG48aNY+/evXh6evLNN9+wYMECevfuTefOnZOTTKb2+YMxjXzDhg3x9PTkrbfeSjU9zJo1axg4cCBgTCOfWsEqpRQxMTHExcURGxtLfHw8JUqUeOoxPiuL3f6rlLIGvgfaAcHAEaXUWhF5/Bv5dxEZ8VjbpkAz4GHhgn1AS6XUKeBLoJ6IhCulFiql2ojIdrMfgLUNtP8MKr0Iq4bDvFbQ4b9Q/w2wQDp4W2srfnjVi67f72for8dYO6IZpZwLmH0/Ws7wxeEvOHfbvKcyaxSpwdiGY9N8X6eRf77SyDdp0oRWrVrh6uqKiDBixAiLZVaw5IykIRAoIhdFJA5YBqSdWexRAtgB+YD8gC0QBlQCAkQkPGm7bUBPs476cVXawvD9UL4ZbBgDv78G0bctsqvCDvmYP7A+MfGJDP3tKA/iMp6oTtNMkTKNuZeXF+fOneP8+fO4ubmxbds2xo4dy969e3FycjKpv6elke/fvz+Qfhr5efPmJf/1HR8fz5AhQ3Bzc6N3796cPfvv358P08jnz5//iTTyly9fTt7OlDTy06ZNw9PTkxdffDE5jXxaHqaRL1CgQHIa+QoVKiSnkX/4eWY0jXzKz3/79u3JaeQ9PT3Zvn178oLH+fPnJ+f3MkVgYCD+/v4EBwdz7do1duzYwZ49e0xunxGWXJBYGghK8TwYaJTKdj2VUi2AAOA9EQkSkYNKqZ1AKKCAWSLir5QqDNRQSlVI6q8bxmBjWQWLw6sr4O8fYNskmPMC9PgRKrxg9l1VK+HIjL6evLnwKO8vP8msfnV1Qaw86Gkzh6yg08jn/TTyq1atonHjxhQsWBCAjh07Jgd8c7PkjCS1b7/Hf6vWARVExB3j7GIhgFKqClATKIMxILVWSrUQkTvAcOB3YC9wGUj1ypVSaqhS6qhS6mh4eHhqm2SMlRU0HQGDt4KNHSzsDDv/C4mZv3D2uNY1SjDOuwYbToUyY3tg+g00zQQ6jfzzlUa+XLly7N69m4SEBOLj49m9e7fFTm1ZckYSDJRN8bwMEJJyAxG5leLpPOBh0eHuwN8iEgmglNoINAb2iMg6jAEIpdRQINXfMBH5EfgRjNl/M3swyUrVhbf2wMb/wO4v4OJu6DkPnMuZbRcAQ1tU4p/rEXyzLYBqJQrS0c3VrP1rzx+dRv75SiPfq1cvduzYgZubG0opvL296dy5c4Y/R5OIiEUeGIPURaAixtNPJ4Haj23jmuLnh8ED4GWMMxQbjNdHtgOdk94rnvTfwoAvUC29sdSrV08s4uQfIlNLi/y3rMjpP83e/YO4BOn2/T6p8fFGOX3trtn717LW2bNns3sIz42BAwfK8uXLs2RfiYmJ4uHhIQEBAVmyP0tJ7fcTOComfN9b7NSWiCQAI4DNgD/wh4icUUpNUUp1SdpsZNJtvCeBkcCgpNdXABeAU0kB6KQYZyIA3ymlzgL7gWki8m8x56zm3huG7QWXKrB8EKwdCXHmu9feztaauf3r4Wxvy5CFRwmPiE2/kaZpWUankTfSha3MITEedk6Ffd+CSzXo9ROUNF9K7dPX7tFrzgFquRZi6dDG5LexNlvfWtbJK4WtckMa+WeV0TTyeUlmClvpQGJOF3fBn2/BgzvQ/lNoONRsa042+IXyzpLj9KpXhi97ues7uXKhvBJItLxJV0i0kARDBu/IqvSicc1JpReNF+OX9oWoW09vY6JO7q6MalOVFceCmb/3kln61DRNMwcdSJ5i1M5RjN83npDIkPQ3fsjBBfr9Dt5fwIXtMLup8c4uc4ynTVU61inJ5xv92fnPkytZNU3TsoMOJGlINCRS2akymy5twmeVD18c/oLbMSauaFcKGg+DwdvBrhD82hW2T8l0NmErK8VXfTyoUbIQI5ecIPBG2vepa5qmZRUdSNJgbWXNmPpj2NBjA50rd2bJuSV0XNmR2b6ziYo3saKhqzsM3QVe/WHvV/CzN9y5nKlx2eezYd7A+uS3teLNhUe5Gx2Xqf40TdMySweSdJR0KMnkppNZ1XUVTUs15YeTP/DSny+x2H8xcYkmfInnc4AuM6HXL3DzPMxpDqdWZGpMpZ0LMLd/PULvxvDOkuPEJxoy1Z/2fMhJaeR1PZLMMbUeydixY6lTp05yJmdL0YHERJWcKvFNq29Y8tISqjhXYdrhaXRZ3YV1F9aRaDAhfUOdHsY1J8VqwMo3YfU7EBv5zOOpV74IU7vXYX/gLT5db9kU91rekJMCSV6XE+qRbNiwgePHj+Pr68uhQ4f48ssvuX//fqb3nRodSDLIrZgb89vPZ27buRTKV4iP9n1E7/W92R20O/3qY4XLw+sbocV/wHcx/NgSQnyf3uYpetcvy5DmFfn14BUW/f1kPh5NS0nXI3lUXq9HcvbsWVq2bImNjQ0ODg54eHiwadOmpx7jMzNl+Xtuf1gqRUqiIVE2XtwoL618SeosqCMD/hogx8OOm9b44h6R6TVEJhcVOTBLJDHxmcaQkGiQgT8fksofbpADgTefqQ8ta6RMQRE6dapcfq2/WR+hU6c+df+XLl2S2rVri4jI5s2bZciQIWIwGCQxMVE6deoku3fvlhUrVsjgwYOT29y9a0zNU758eQkPD0+z7xs3bkiZMmXk4sWLIiJy69YtEREZMWKETJo0SUREtm/fLh4eHiIi8ssvv8g777wjIiJ16tSR4OBgERG5c+eOiIhERUXJgwcPREQkICBAHv4b3rlzpzg5OUlISIjExMRIqVKl5JNPPhERkW+//VZGjRolIsYUKR06dJDExEQJCAiQ0qVLy4MHD2Tnzp3SqVMnERH58MMP5bfffkveb9WqVSUyMjLV4/vll1+kZMmScvPmTYmOjpbatWvLkSNH5NKlS1K3bl0RMaZKqVSpkty8efOR/TxsX7p06eTPJa3P/+zZs+Lj4yNxcXEiIjJ8+HBZuHChiIi8+eabcuTIERERcXJyemR8zs7OT4x58+bN0rRpU4mKipLw8HCpWLGiTJ8+Pa3/hZlKkWLJpI15npWywruiN23Kt2HV+VXMPjmbARsH8GKZF3nX612qFa6WduOKzY1rTtaMgM0fwYWd0G02FCyWoTFYWylm9K1Ljx8OMHzxMda+8wLlitpn8si0vC5lPQyAyMhIzp8/T/PmzXn//fcZO3YsPj4+NG/e3KT+nlaPZOXKlUD69Uj69OlDjx49AGM9khEjRuDr64u1tTUBAf9mQnpYjwR4oh5JyoSSptQjWbt2LdOnTwdIrkeS1qLRh/VIgOR6JKNHj06uRxIWFvZM9Ujg38/fz88vuR4JwIMHD5JTxM+fPz/VftPSvn17jhw5QtOmTSlWrBhNmjTBxsYyX/k6kJiBrZUtfar3waeSD0vOLeHnUz/Ta20vOlfuzNueb1O6YOnUG9oXgVcWw5H5sHm8cc1Jj7lQuXWG9l/Izpb5A+rT9fv9vLnwCH++3RRHO1szHJlmKSU/+ihb9y+6Hkmer0cCxlNt48ePB6Bfv34Wywemr5GYkb2tPYPdBvNXj78YVHsQmy5tovOqzk9fg6IUNBwCQ3caA8tv3WHLBEjI2G29FVwcmP2qFxdvRjF6mS+Jhryf+kbLGF2P5PmqR5KYmMitW8bMGn5+fvj5+SXP3sxNz0gswNnOmTH1x9CvZj/mnJzDknNL+PP8nwyqPYgBtQfgYJtKbYUStWHITtgyHg7MgMt7oedPULSyyfttWsWFSZ1rMWHNGf63+RwfdtR5nbR/6Xokz1c9kvj4+ORTk4UKFWLRokUWO7WlkzZmgYv3LjLrxCy2XtlKEbsiDHUfSu9qvclnnUaV4LNrYe27YEiATl+Dx8sZ2t/4VadYfOgqX/fxoIdXGTMcgWYOOmlj1hk0aBA+Pj706tXL4vsyGAx4eXmxfPnyXJ1KXidtzOEqOVXi6xe/Nn0NSq0uxgvxJd1h1VBjRuFY09OhTOpSm8aVijBu5SmOX019oZKmaZmn65EY6RlJFhMRDoYc5Nvj3+J/258qzlUY7TWaFmVaPHmx0pAIe6bD7mngXN5Y56R0PZP2cycqjq7f7yc6LpF17zbD1amABY5Gy4i8MiPR9UjyJl2PJB05KZA8ZBADWy5vYeaJmVyNuErd4nUZ7TUarxJeT2585SCsHAyR16HNJ9DkXbBKfzIZEBZBjx8OUMHFnuVvNaVAPl0QKzvllUCi5U361FYu9HANyupuq5nQeAJBEUEM3DSQEdtHEHDnserB5ZvA8H1Q/SXY+gks6gERYenuo1oJR2b09eRMyH3eX3Ey/ZX3msXp/wdaTpTZ30sdSLLZwzUoG7pvYJTXKI6HHafX2l58tPcjrkVe+3fDAoWhz6/Q+Tu4+rdxzcn5ren237pGCcZ612CDXygzdwRa8Ei09NjZ2XHr1i0dTLQcRUS4devWE3e1ZYQ+tZXD3Iu9x0+nfmLJuSUkSiIvV3+ZIW5DKFogxWrZG+dgxRtw4ww0fgfaTgSb/Gn2KSL83x8n+fPENWa/6kVHN9csOBLtcfHx8QQHBxMTE5PdQ9G0R9jZ2VGmTBlsbR9dyKyvkaSQmwLJQ9ejrjPn5BxWBa7CztqOgbUHMrD2wH/XoMTHwNYJcPhH491dvX4Blypp9hcTn0jfeX9zLjSCFcObULuUUxYdiaZpuZUOJCnkxkDyUMo1KIXzF2ao+1D6VO/z7xqUc3/BmncgIRZe+hI8+xlXy6fiRkSblRANAAAgAElEQVQMXWftRwFrRrxAMce0ZzGapmn6YnsekXINStXCVfniyBd0XtWZtRfWGteg1HjJuOaktBesedtY6yTmXqp9FXe0Y96A+tyOjmPYomPEJmQ87YSmadrjLBpIlFLeSql/lFKBSqlxqbw/SCkVrpTyTXoMTvHe/5RSZ5RS/kqpGSppkYVSqq9S6pRSyk8ptUkp5WLJY8gpkuugtJuLU34nxu8bT691vYx1UBxdYcAaaD0BzqyGOS9A0JFU+6lT2omventy7Modxq86rS/8apqWaRYLJEopa+B7oCNQC+irlKqVyqa/i4hn0mN+UtumQDPAHagDNABaKqVsgO+AViLiDvgBWV+vM5sopWhaqinLfJbxZcsviUuMY8SOEQzcNJDj4SehxfvwRlLhmp87GOvEp7JyvpO7KyPbVGXFsWB+2ncpi49C07S8xpIzkoZAoIhcFJE4YBnwZIrK1AlgB+QD8gO2QBigkh4OSTOUQkCIuQee01kpK7wr/LsGJTgi+N81KAWdYdg+qNUVtk+B37rB/dAn+hjdpiod65Tkv3/5s/OfG9lwFJqm5RWWDCSlgaAUz4OTXntcz6TTVCuUUmUBROQgsBMITXpsFhF/EYkHhgOnMAaQWsBPqe1cKTVUKXVUKXU0PDzcbAeVkySvQenx2BqUI18Q3GEKdP0ego8a15z8s/GRtlZWiq/6eFCjZCFGLjlB4A3Tc3lpmqalZMlAktqtQ4+fkF8HVEg6TbUNWAiglKoC1ATKYAw+rZVSLZRSthgDSV2gFMZTWx+mtnMR+VFE6otI/WLFMlZ1MLcpYFOAwW6D2dhzI4PqDGLLlS10XtOFafHB3Bq0FpxKw9JX4K//GG8bTmKfz4Z5A+uT39aKNxce5W50xmqgaJqmgWUDSTBQNsXzMjx2GkpEbonIw+xv84CHGQm7A3+LSKSIRAIbgcaAZ1K7C0n1hP8AmlruEHIXp/xOjKk3hvXd19O1cleWnVvGS7tG8EPDPkQ2HAKH58L8NhD+T3Kb0s4FmNu/HqF3Y3hnyXHiEw3ZeASapuVGlgwkR4CqSqmKSql8wCvA2pQbKKVSLrHuAvgn/XyVpIvrSbOQlknvXQNqKaUeTjHapWhjdhHbtvHg1Klcd2dTSYeSTGo6iT+7/kmz0s2YfWoeL90/zKJW7xIXcR3mtoRjCyDpuOqVL8LU7nXYH3iLz9afzd7Ba5qW61isQqKIJCilRgCbAWvgZxE5o5SaAhwVkbXASKVUFyABuA0MSmq+AmiN8VqIAJtEZB2AUmoysEcpFQ9cSdHG3OPn+tT/khAaim3p0jh26EChjt7Y1amTam3qnOjhGpTTN0/z7bFv+eLyGn6rWJF3og10WjcK6ws7jLm7ChSmd/2yBIRFMG/vJYo45Ofd1lWwssodx6lpWvbSK9ufIvHePSK27+D+5k1EHTgI8fH/BhXvDti5ueWaoCIiHAw9yLfHkuqg5CvMqOBAWtoUQfWcD+Uak2gQxvzhyxrfEFrXKM7XfTxwtk+jiqOmaXmeTpGSgjlSpKQaVEqV+jeouLvniqBiEANbrmxh5vGkOigJMDr8Bl5N/g+a/x+irPj14BU+23CW4o52fP+qF55lnbN72JqmZQMdSFIwd66txPv3idi+g4hNm4g8cADi47Ep5Uqh9klBxcMjxweVeEM8q86vYo7vD4TH3KJl9ANG2lWgWo8F4FQG36C7vLP4ODciYhj/Uk0GNq2Q449J0zTz0oEkBUsmbUy8f5+IHTuI2LSZqP37kfh4bFxdKdS+PY7eHSjg4YEyoZphdnmQ8IDF/ov52XcOkYkx+DyI582mE6js8Rp3o+P4vz9Osv3cDTq5uTKtpxuOdrbpd6ppWp6gA0kKWZX9NzEigsgdO7i/aTNR+/YZg0rJkhTq0B7HDt4U8My5QeVe7D1+OvIVSwJXEaugjn1purm9Tofy3iw9dJPpW/6hXBF7vu/nRa1ShbJ7uJqmZQGzBRKlVGUgWERilVIvYsx/9auI3DXLSLNAdqSRT4yIIHLnTmNQ2bs3Oag4tm9HIW9vCnh65sigcuvuZTb8+SqrE29yPl8+8lnlo3W51tRwaMWczTbcf5DIp13r0KdB2fQ70zQtVzNnIPEF6gMVMN7KuxaoLiIvmWGcWSK765EkRkY+GlTi4rApUQLH9u0p1DEHBpW4KGRpX/yvHWBNHW82RF/mXuw9itoVwyqqPpcu16KnW10+7VqHAvmss3u0mqZZiDkDyXER8VJKfQDEiMhMpdQJEalrrsFaWnYHkpSeGlS8O1Cgbt2cEVTiY4zlfP/ZQFzrj9ld3pPVgavZd20fBjGQGF2OIvICP3R7HfdSJbN7tJqmWYA5A8kh4FtgPNBZRC4ppU6LSB3zDNXyclIgSckYVHYZbynekxRUihf/N6h4eWVvUEmMh9XD4dRyeGEMtPmE8Ac3WX9xPUvOruT6gyuIwZa6RZvzToO+NCzZECuVA4KgpmlmYc5AUgsYBhwUkaVKqYrAyyIyzTxDtbycGkhSSoyMInLXLiI2byJyz14kNhabYsUeDSrW2XAayZAI69+D4wuh4VvgPQ2srBARdl05xoRtv3DX6jDKOoaS9q50rdKFrpW7UraQvoaiabmdRe7aUkoVBsqKiF9mBpfVckMgSSkxMorI3buI2LSZyD17kNhYrIu5UKid8ZZi+3r1sjaoiMCWj+HgLPB8FbrMBCvj/uMTDXy+0Y9fT/5FkZInibHxRxDqlahH18pdaV+hPQ62Dlk3Vk3TzMacM5JdGBMq2gC+QDiwW0TGmGGcWSK3BZKUDFFRRO7ezf1Nm4ncvTtFUGmHYwdv7OtnUVARgV3TYPc0qNUNeswDm3/Tp2w+c533l58E67u81CSEsxE7uHz/MgVsCtCufDu6VelGvRL19KkvTctFzBlITohI3aR66mVFZKJSyi+phkiukJsDSUqPBJU9e5CYGKxdXHBs15ZC3h2zJqgcmGmcnVRtD31+BdsCyW9dvRXN8MXHOBNyn6HNK9K+XgzrL65l0+VNRMVHUbpgabpW7kqXKl0oXTC1GmeapuUk5gwkp4D2GItOjReRIzqQZD9DdPSjM5WUQaWDN/YN6lsuqBz9xXjdpMIL0Hcp5HdMfismPpFP159l8aGrNKhQmJl9vXByELZf3c7qwNUcDj2MIDQs2ZBuVbrRplwb7G3tLTNOTdMyxZyBpDcwAdgvIsOVUpWAL0Wkp3mGanl5MZCkZIiOJnLPnn+DyoMHWBctmjRT8ca+fn2UjZkrBvj9AauGQam68NoKKFD4kbfX+F7jwz9PUcDWmu9eqcsLVV0ACIkMYe2FtawJXENwZDAOtg50qNCBrpW7Urd4XZ3PS9NyEJ0iJYW8HkhSMgaVvdzfvInIXUlBpUgRHNu1o5B3B+wbNDBfUPFfDyteB5dq0H8VFCz+yNuBNyIYvug4geGRjGpTlXdbV8U6qcaJiHAs7BhrLqxh8+XNPEh4QDnHcnSt0pUulbtQ0kGvTdG07GbOGUkZYCbQDGORqX3AKBEJNsdAs8LzFEhSMjx4YAwqmzY+GlTatjUGlYYNMx9UArfDsleNdeEHrAGnMo+8HR2XwMerTvPniWs0r+rCNy974lIw/6PbxEez5coW1gSu4WjYURSKxq6N6ValG63LtcbOxi5zY9Q07ZmYM5BsBZYAvyW99Brwqoi0y/Qos8jzGkhSehhUIjZvImLXbiQ6GuvChXFs2xZH7w44NGr07EHlykFY0gfsnGHgGihS6ZG3RYRlR4KYuPYMhe1tmdXPiwYViqTaVdD9INZeXMvawLWERIXgaOuId0VvulXphptL7ikkpml5gVlzbYmIZ3qv5WQ6kDzKEBND5J49RGzaTMSuXcag4uyMY7u2OHbwxqFJ44xfqA85Ab/1AOt8MGA1FK/5xCZnQu7x9uLjBN95wFjv6gxpXinNwGAQA0euH2F14Gq2XdlGTGIMFZ0q0rVyVzpX7kxx++KpttM0zXzMGUi2AQuApUkv9QVeF5E2mR1kVtGBJG2GmBgi9+41Ln7cuRNDdDQOTZtQesYMrAsWzFhnN/zh167G1Cr9/zReiH/M/Zh4xq7wY+Pp67StWYKvenvgZP/0GieRcZFsvryZNRfWcOLGCayUFU1LNaVblW60KtuKfNa6HLCmWYI5A0k5YBbQBOM1kgPASBG5ao6BZgUdSExjiInh3qpVXJ/6X/JXrUrZuXOwLZ7Bv/xvXYBfu0HMXXh1OZRr/MQmIsIv+y/z37/8cXW244d+9XAr42RS95fvXTbe9XVhDTeib1AoXyFeqvgS3ap0o1bRWvrUl6aZkUXv2lJKjRaRb59pZNlAB5KMidy7l+BRo7EpXJiy8+aRv1LFjHVwL9g4M7kfAq8shsqtU93s+NU7jFh8nJuRcUzwqclrjcubHAgSDYkcCj3E6sDVbL+6nThDHFWcq9CtSjc6VeqESwGXjI1Z07QnWDqQXBWRcs80smygA0nGPTh1mqBhwyAhgTJzZmNfN4NVAyJvwG/d4WYA9F4ANTqlutmdqDje+8OXXf+E09mjFJ/3cKNg/oxd9L8fd59NlzaxJnANfjf9sFbWNC/dnG5VutGiTAtsrXV5YE17FpYOJEEikm56V6WUN/AdYA3MfzxjsFJqEPAlcC3ppVkiMj/pvf8BnQArYCswCigI7E3RRRlgkYiMfto4dCB5NnFXr3J1yBASrodR+puvcWyd+swiTdG3YXEvCPGF7nPBvXeqmxkMwuzdF/hqyz9UcHFg9qv1qF7SMdVt03Ph7gXWXFjDugvruPngJoXzF6ZTpU50rdKVGkVqPFOfmva8yvYZiVLKGggA2gHBwBGgr4icTbHNIKC+iIx4rG1TjAGmRdJL+4APRWTXY9sdA94TkT1PG4sOJM8u4dYtgoYNJ+bMGUp+8gmFX3k5Yx3ERsDSvnB5H/h8A/VfT3PTAxduMnKpL5Gx8XzWzY1e9cqkuW264zYkcCDkAKsDV7MraBfxhnhqFKlBtyrdeKniSxS2K5x+J5r2nDM1kKSZilUpFaGUup/KIwIoZcIYGgKBInJRROKAZUBXE8cvgB2QD8gP2AJhj42vKlCcR2compnZFC1K+YULcGj+AtcnTSJ8xgwy9MdHfkfjRfcqbWH9aGPSxzQ0rezCX6NewLOsM+8vP8nYFX7ExCc+27itbGhRpgVfv/g1O3rv4MOGH2KlrJh2eBqtl7fmvZ3vJQcYTdMyJ81AIiKOIlIolYejiJhyErs0EJTieXDSa4/rqZTyU0qtUEqVTdr3QWAnEJr02Cwi/o+16wv8Ls9DjpdsZmVvT9nvv8epZw9u/jCb0AkTkIQE0zuwLQCvLIFaXY2Zg3dNM6alT0VxRzsWvdmId1pV5vejQXT7fj+XbkZlavzOds70q9mP331+Z2WXlfSr0Y/jN47z7o53abe8HdOPTCc8OjxT+9C055kli0OkdvvN498e64AKSZmEt2HMMIxSqgpQE+M1kNJAa6VUi8favsK/a1ue3LlSQ5VSR5VSR8PD9ZdEZikbG1w/+wyXt4dzb8VKgt55B0N0tOkd2OSDnj8bC2Pt+twYUNIIJjbWVnzQoQa/DGrA9fsxdJ65j79OhZrlOKoVrsYHDT5gW+9tzGg1A49iHiz2X0zXNV1ZHbg6Y7MtTdMAywaSYCDlBfkyQEjKDUTklojEJj2dB9RL+rk78LeIRIpIJLARSF6QoJTyAGxE5FhaOxeRH0WkvojUL1asWOaPRkMpRbGRIyk5aRJRe/dxZeAgEm7fNr0DaxvoMgsaDjVWW1w/2ljKNw2tahRnw8jmVClekLcXH2fS2jPEJRjMcCRga2VLq3Kt+K71d6zquoqqzlWZsH8Cw7cNJyQyJP0ONE1LZslAcgSoqpSqqJTKh3EGsTblBkop1xRPuwAPT19dBVoqpWyUUrZAyxTvgfG0VpqzEc2yCr/yMmVmziA2IIDLffsSdzUDa1OtrKDj/+CFMXBsAax6y7gSPg2lnQvwx1tNeKNZRRYcuEzvuQcJvpOBmZAJKjhV4BfvX/io0Uccv3Gc7mu6s+zcMgxinqClaXmdxQKJiCQAI4DNGIPAHyJyRik1RSnVJWmzkUqpM0qpk8BIYFDS6yuAC8Ap4CRwUkTWpei+DzqQZCvHNm0ot+AXDHfvcblvPx6cPmN6Y6Wg7URo8wmcWg5/DISE2DQ3z2djxSedazH7VS8u3ojEZ+Y+dpwLS3P7Z2GlrOhboy+ruq7Co5gHUw9N5Y3Nb3Dl/hWz7kfT8iJTUqRE8OS1jXvAUeD/ROSihcZmNvr2X8uJvXiJoMGDSbh7lzLffUfB5i9krINDc2Hjf6BSK+Mq+HwOT9388s0ohi8+jn/ofd5+sTJj2lXDxtq8fw+JCKsDV/PlkS+JM8Txbt13ea3ma1hbWbiMsablMObMtTUZ47WNJRgvoL8ClAT+AYaLyIuZHq2F6UBiWfE3bhA09C1iAwNx/exTnLt1y1gHJxbB2nehTEN49Q+we3rerZj4RCatPcOyI0E0qliEmX3rUryQ+WuW3Ii+wacHP2VX8C7cXNyY0nQKVQpXMft+NC2nMmcgOSQijR577W8RaayUOikiHpkcq8XpQGJ5iZGRBL/7LtEH/6bYe+9RdOiQjCVQPLMKVg6GErXhtVXgUDTdJiuPBfPx6tM45LdhRl9PmlY2f34tEWHjpY18fvhzIuMjGeY+jDfc3sDWSqdd0fK+TC9ITMGglOqjlLJKevRJ8Z6+V1IDwLpgQcrNnUshHx/Cv/mGsE8/RRIzsJiwdnd4ZSmE/wMLXoKI6+k26VmvDGtGNMOpgA2vzT/EzO3nMRjM+yuplOKlSi+xuutq2pZryyzfWfRd35ezt86m31jTnhOmzEgqYcyX1STppYPAexjzY9UTkX0WHaEZ6BlJ1hGDgRtffcXtn37GsV07Sn35P6zsMnDa6dJeWPoKOBQzlu4tXD7dJlGxCXy06hRrfENoWa0Y37zsSREHy9Qo2X51O5/9/Rl3Yu7wep3XGeYxjPzW+dNvqGm5kEVzbeU2OpBkvdu//krY59Mo4OVF2e9nYe3sbHrj4KOwqAfkK2gMJi5V020iIiw+dJUp685StGA+ZvXzol55y+TTuhd7jy+PfMmaC2uo6FSRKU2n4Fk81xQM1TSTme3UllKqjFJqlVLqhlIqTCm1Uin17Nn0tOdCkQEDKP31V8T4+XH51deID8nAIr8y9WHQBuMtwb90hOun0m2ilOK1xuX58+2m2FgrXp57kJ/2XbLISnWn/E589sJnzGk7h5iEGAZsHMAXh78gOt6861s0Lbcw5RrJLxgXEpbCmK5kXdJrmvZUhTp2pOz8+STcuMHlV/oS888/pjcu6QZvbDLWgF/QCYKOmNSsTmkn1r/bnNY1ivPp+rMMX3Sc+zGWSczYrHQzVnVdRZ/qfVjkv4iea3tyKPSQRfalaTmZKYGkmIj8IiIJSY8FgM45opnEoVFDyi9aBEpx5dXXiPo7A1+0LlXh9Y1QoLCx4uKlp1YLSOZUwJa5/evxcaeabPMPw2fGPk5fu/eMR/B0DrYOfNz4Y37u8DNWyorBWwYz+eBkIuIiLLI/TcuJTAkkN5VSrymlrJMerwG3LD0wLe+wq16NCsuWYlOyBEFDhnD/r79Mb1y4PLy+CZzLwuLeELDFpGZKKQY3r8SyoY2JSzDQY/YBlhy6arGkjA1KNmBFlxUMrDWQP8//Sfc13dkTbFrg07TczpRA8gbGlCTXMaZ07wWkXZ1I01Jh6+pKhcWLsfNw59qY/+PWggWmNy7kCoP+gmLVYVk/45oTE9WvUIQNI1+gUcUifLTqFGP+OEl0XAZS4GdAAZsCvN/gfX7r+BsFbQvyzvZ3+GjvR9yLtcxsSNNyimetkDhaRL61wHgsQt+1lXMYYmMJ+eA/RGzZQpHXX6f4B++jrExMcRJzDxb3geDDxizCdV81eb+JBuH7nYF8sy2AKsUK8sOrXlQt8WzlfE0RlxjHXL+5/HzqZ5zyOzG+8XjalW9nsf1pmiWYc0FiasY8YzvtOWeVPz+lv/mawq++yu1ffiHkg/9giIszrbGdE/T/Eyq2hDVvw6EfTd6vtZViZJuqLHqzEXei4+gyaz+rT1x7xqNIXz7rfLxb912W+iyluH1xxuwaw5hdY7j54KbF9qlp2eVZA0kGcl9o2qOUtTUlPh5Psf8bw/0NGwga+haJESZenM7nAP1+h+qdYOMHsPerDO27WRUXNoxsjltpJ0b/7suHf5565nK+pqhRpAaLOy1mlNcodgXtotuabqy7sE4X0NLylGcNJPpfgZYpSilchgyh1BfTiD56lCuv9Sc+7IZpjW3yQ5+F4NYbtk+BbZPSrLaYmhKF7FgypBHDWlZm6eGr9Jx9gN0B4cQnWqb+iK2VLYPdBrOi8wrKFyrPR/s+YsSOEVyPSj8NjKblBmleI0kjfTwYZyMFTKzbniPoayQ5W+S+/VwbORIrZyfKzZtH/sqVTWtoSIQNSQWyGg4F7y+MhbMyYNvZMD5YcZI70fEUtrfFu44rnd1daVSpKNZW5p94JxoSWXJuCTOOz8DGyoYx9cfQq2qvjCW41LQsolOkpKADSc734MwZgt4aBvHxlJk9G3uvuqY1FDHWfz84y1gPvvMMY0nfDIiJT2R3QDjr/ULZ7h9GdFwiLgXz85JbSXzcS1G/fGGszBxUgu4HMengJA5fP0yjko2Y2HQiZR3Lpt9Q07KQDiQp6ECSO8QFBRE0eAjx169T+qvpOLZta1pDEdj9Bez6HGp1gx7zwObZkjY+iEtkx7kbrPcLYce5G8QmGChZyI6X3Fzx8XClbllns80eDGJg5fmVfHX0KwxiYGTdkfSt0VcX0NJyDB1IUtCBJPdIuH2boGHDiTl9mpKfTKDwK6+Y3vjALNgyHqq2hz6/gm2BTI0lMjaB7f5hrDsZyp6AcOISDZR2LoCPuys+7qWoU7qQWYLK9ajrTD44mX3X9uFZzJPJzSZTyalSpvvVtMzSgSQFHUhyF0N0NNfG/B+Ru3ZRdNhbFBs1yvQv7KO/wPr3oMIL0Hcp5DfPWpF7D+LZejaM9X4h7Dt/kwSDUKGoPZ2SgkqNko6ZCioiwvqL65l2eBoxCTEM9xzOoNqDsLHKNZcitTxIB5IUdCDJfSQhgeuTJ3N3+QqcevTAdfIklK2JVQn9lsOqt6CUJ7y6AuyLmHVsd6Li2HzmOuv9Qjlw4SYGgSrFCybPVKoUL/jMfd98cJOpf09l29Vt1CpaiylNp1C9SHUzjl7TTKcDSQo6kOROIsLN73/g5qxZOLRoTplvvsHKwcG0xv7rYcXr4FIN+q+CgsUtMsabkbFsPH2d9SdDOHz5NiJQo6QjnT1K4ePuSvmiJo73MVsub2Hqoancj73PYPfBDHUbiq21Lu+rZS0dSFLQgSR3u/PHH1yfNBm7WrUoO3cONkXTr+cOwIUdsOxVKFTKWCDLybJldMLux/DXqVDW+4Vy7ModANxKO+Hj7kond1fKFLbPUH93Y+7yxZEvWH9xPVWcqzCl6RTcirlZYuialiodSFLQgST3i9ixk2tjxmBTvDjl5v1IvvLpl+AF4MpBWNIH7JxhwGooauIalUy6dvcBf/mFst4vhJPBxqSNnmWd6exRik5urpR0Mr388O6g3Uz5ewo3H9xkQK0BvOP5DnY2GShfrGnPKEcEEqWUN8Z679bAfBGZ9tj7g4AvMdZ/B5glIvOT3vsf0Anj6vutwCgREaVUPmAW8CJgAMaLyMqnjUMHkrzhga8vQcOGg5UVZefOoYCbiX+dh5yA33oYi2QNWA3Fa1p2oI+5eiua9adCWH8ylLOh91EKGpQvgo+HKx3ruFLMMf2a7xFxEXx19CtWnl9J+ULlmdx0MvVK1MuC0WvPs2wPJEopayAAaAcEA0eAviJyNsU2g4D6IjLisbZNMQaYFkkv7QM+FJFdSqnJgLWIfKyUsgKKiMhTM+HpQJJ3xF68RNCQISTcvk2Z776lYIsW6TcCuOEPv3aDxDhj4sdSJi54NLML4ZFsSJqpBIRFYqWgcaWi+LiXwrtOSYo4PH39y9+hfzPpwCSuRV7jleqvMLreaBxsn+06jKalJycEkibAJBHpkPT8QwAR+TzFNoNIPZA0wTjreAFjSpY9QH8R8VdKBQE1RCTK1LHoQJK3JISHc/Wtt4j9JwDXTz/FuUd30xrevggLu0LMXej3B5RvYtmBpuOf6xGs9wthvV8ol25GYW2laFbFBR93VzrUKomTfeoX16Pjo5lxYgZL/Jfg6uDKxKYTaVqqaRaPXnse5IRA0gvwFpHBSc/7A41SBo2kQPI5EI5x9vKeiAQlvTcdGIwxkMwSkfFKKWfgFLAc46mtC8AIEQl72lh0IMl7EiOjuDZyJFEHDlBs9CiKvvWWaes47l0zlu29Fwx9l0Dl1pYfbDpEhDMh99lwyjhTCbr9AFtrRYuqxfDxcKVtzRI42j0ZVE7cOMEn+z/h8v3LdK/SnfcbvE+hfIWy4Qi0vConBJLeQIfHAklDEXk3xTZFgUgRiVVKDQP6iEhrpVQVjNdWXk7adCswFjiLMej0EpGVSqkxQF0R6Z/K/ocCQwHKlStX78qVKxY5Ti37SFwcIR9/zP2163Du+wolP/4YZW1CepHIcPitG9wMgF6/QE0fyw/WRCKCX/A91vuFsMEvlJB7MeSzsaJV9WL4uJeiTc3i2Of7d5FibGIss31ns+DMAorYFWFC4wm0KtcqG49Ay0tyQiBJ99TWY9tbA7dFxEkp9QFgJyKfJr33CRCD8bpJJOAoIgalVFlgk4jUftpY9Iwk7xKDgfBvvuHWvPkUbNuG0tOnY2Vnwh1N0beNNeBDTkD3OeDex/KDzSCDQTgRdId1J0PZcCqU8IhYCtha07pmcTq7u/Ji9eLY2RoD55lbZ/hk/ycE3AmgY8WOjLdAlLQAACAASURBVGs4jiJ25l2IqT1/ckIgscF4uqoNxruyjgD9RORMim1cRSQ06efuwFgRaayUehkYAnhjPLW1CfhWRNYppZYBP4rIjqRTY53+v707j6+6vvM9/vqcJCd7QhbCkoUtLoCCCgbUquDCphJcWsW2ir3VqbedzqOdLtOZzr2OvTN22t7HzL0P7czVjkvtiNO6EFACLiWCuABWkE3ZTULYE5KT7ayf+8fvJBxCKgnJOSchn+fjkQdn+Z3f+X455Lz5fH+/3/erql/+orZYkJz/6p//HUf+6Z9Ivewyin79BIk5OWd/kdcDSxfDgXeh6ErnbK4Rk6FgkvOT3sPrVWIgGFI27K/ntU/qWLXtMCdafKS7E7h50ghunTKaay/MxyUhfrPtNzz5yZNkJmXykxk/Yd7YeTZFvTlncQ+ScCMWAP+Kc/rv06r6jyLyKLBJVZeLyGPAQiAA1AMPq+qn4erk1zhnbSlO1fH98D7HAM8Dw3CGuR5Q1eovaocFydDQtGo1dT/6EUlFRZQ89SRJhYVnf5G/zZk5uGYDHNnuHIjvkDHiVKiMmOQEzfCJ4O7dhYX9LRAM8f6+E7y25RCrth+msc1PZkoicyeP5NYpoxiR38Cj7z/CthPbmF08m5/O/CkFadG5st+c3wZEkAwUFiRDR+vGjdR8+zu4kpMpfupJUi6+uOcvVgXPYTi63Tld+MgO5/axzyDQHt5IIHfcqYDpqGJyJ/R6HZT+4AuEWL/nOCs+qePN7UfweAPkpCUxZ/JwUvLW83rts7hdbn545Q9ZVLrIqhPTKxYkESxIhhbv7t1UP/gQIY+HoiceJ33mzL7tMBSE+v1wdIfzcyQcNPV7QcPL8ya4If+icOUSUcVkFUKMvrzb/UHWhhfoeiu8QFfusJNkFb3KieBnXDXqah65+n8yOmN0TNpjBj8LkggWJEOP/9Ahah56CO+Bzxn92GNk33pLFN6kzTnzq6Ny6ahiPHWntknODlctXQImtQfHcPqgzRdkzWcdC3QdJpjxHikFq0h0ubj3wof5/oz7bQEtc1YWJBEsSIamYFMTtf/927Ru2kTBj35E3jceiM0bt9bDsU9PVS5HdzgB4208tU3mqNOHxgomwfCL+rwYV3davAHe2nmEl7Zs5U8tT+JK343LO57RrpsoSb2CkRnZ5Gckk5eRTF6Gm/wMN3npzu2M5EQbDhvCLEgiWJAMXSGvl7of/w2eVavIvf9+Cn78I8Tlin1DVKGpLmJ4rOP4yy4Iep1txAW547sc4J/kPNZP1UNjm49fvPs8lXX/gR8PaCK0TaC96WICnoloYNhp27sTXeSnuztDJi89mfwMdzh4wo+nO/dz0924E+Pwd2uixoIkggXJ0KahEEce+zkNzz9P1oL5jPr5z3G5z21N934XDDhTt3QOjXUcf9mHc8IikJjiVCsFXYbHMked8/GXQCjA5qObqaqpYk3NGqo9zomPYzMvZGLWTIpTrsQdLKK+xc/xZh8nWrycaPZxotnL8WYfvmCo2/1mpSSeCpn0U2HTGT7pp+5npSThclm1M5BZkESwIDGqSv3TT3P0l78irayMoiceJyGzf5bhjQpfqzM81jE01lHFNB8+tU3KsNMrl46hstRhf36/3VBV9jftp6qmiqqaKjYf3YyiFKQVMLt4NrOKZ1E2sgx3grtz+2ZvwAmWFi/HPF2CpsX503neR0Orj+6+ZhJdQm5EsORFVD756cnkZ54Ko/yM5M6LL03sWJBEsCAxHRqXL6fub/+O5PHjKX7qSZJGjIh3k3qntT5iaKzjZyd4m05tk1V4ZsDkXwhJPVvD5ETbCdYdXEdVTRXv1b1HW6CNtMQ0rim8hlnFs7i28FpyUnp+skAgGKKh1d8ZNsc7Q8bL8XAIRVY9rb5gt/tJdyecMcQWWfnkZyR3VkM5aW4SrNrpMwuSCBYkJlLz+vUc/Mvv4srOpuSpJ0kuLY13k/pG1ZmE8rTjLzuc619CfmcbSXAW9eoIlpGXwLjrIPmLqzJv0MuHhz7srFaOtR3DJS4uG35ZZ7UyNntsv3an1RforGY6KpvjERXPiRYfxzzOn/UtPoKhM7/DRCA37fSgGT0sldumjObSoux+be/5zIIkggWJ6ap9xw6q/+IvUJ+f4l8/Qdq083CRqKAfTuw9M2AaDgDqHHu5YA5ccgdcMPesV+yHNMTOEztZU7OGqpoqPmv4DICxWWM7Q2Xq8KkxPa04FFIa2/ynqprmiAonovI50eyj9mQbvkCISwuzWVxWwsLLRpORHPuLSAcTC5IIFiSmO77aWmq++SD+ujpyFt9D1oIFpEyZcv6f7uprcSar3FEB25dBy1FISoeL5sHkO6D0ph4Ng9U113UerN90eBMBDZCTnMO1Rdcyu3g2V4++mrSk+E4nE6mxzU/F5oO88GE1nx72kOZOoPyy0SwuK+HSwuzz/3M/BxYkESxIzJ8TaGjg8D88SvPbb6N+P0mFhWQtmE/W/PkkT5x4/n+5hILw+XrY9grsXA6tJyA5Cy5a4FQq42dD4tnPcPP4PKw/uJ41NWtYd3AdHp8Ht8tN2agyZhfP5vqi6xmRPjCOR6kqm2tOsnRDNSu2HKLNH2Ty6CwWl5VQftnobtd+GaosSCJYkJizCXo8eN56m6bKlbS89z4EArjHjCHrlgVOqFxwQbybGH3BAOx/B7a/AjtXQHujc2bYxFth8u0w7npIOPuXrD/k5+MjH7OmZg1ratZwsPkgAJPzJjOreBazi2dzYc6FAyKkm9r9VGyu44UPq9l5qInUpAQWTh3N4hklTC2yKsWCJIIFiemNQEMDnjffpKmyktYPN0AoRPIFF3RWKu6xY+PdxOgL+GDfGqdS+fR18HkgNRcmLXSGv8Z+qUcXSaoqe0/uparWGQLbemwrijIqfRSzimcxq3gWV464kqQeBFQ0dSwotnRDNcu31NHqCzJxVBb3lhVTfnkhWUO0SrEgiWBBYs5V4PhxmlavpqmykrZNHwGQMmkSWQvmkzlvPu6iHkxVP9j522HPW06l8tkq8LdAegFMKneGv4pnQg9nCzjedpy1tWtZU7OGD+o+oD3YTnpSOl8q/FLnqcXZyfE9q8rT7mf5FqdK2V7XREqSi9umOFXK5cXDhlSVYkESwYLE9Af/4cM0rVpF08pK2j/5BIDUqVPDoTJv8F2Tci58rbB7NWx/FXatdqbXzxwNkxc5lUrR9B5fbd8WaDvt1OIT7SdIkASuGHEFs4qcIbDirOIod+iLba1t5IUN1SzffJAWX5CLR2ayuKyERZcXkp16/lcpFiQRLEhMf/PV1tJUWUnTykq8O3eCCGnTppG5YD5Zc+eSmDdwVleMGm8z7FrlDH/teROCPsgucULlkjtg1GU9DpWQhth2fFvnWWB7Tu4BYEL2hM4hsEvzL43bjMXN3gDLN9exdEM1Ww82kpLk4pZLR3PvjGKuKMk5b6sUC5IIFiQmmrz794dDZSW+PXvB5SJ95gyyFiwg86abSBjWuylLBqX2Rvh0pTP8tfePEAo4k01Ovt2pVEZM7tW8YDWeGt6peYeqmio2HdlEUIPkpuRyfdH1zCqexcxRM+N2avHW2kaWbqym4mOnSrlwRAaLy0q44/IistPOryrFgiSCBYmJlfZdu2hauZKmykr8n1dDYiLp11xN9oIFZNx4IwkZGfFuYvS11sOnrzmVyv61oEFnipbJdziVyvCLerW7Rm8j6w+up6qminUH19HsbyY5IZmZo2Yyq3gW1xddz/C04VHqzJ/X4g2wYotTpWypbSQ50cUtl45i8YwSpo85P6oUC5IIFiQm1lSV9h07OkMlUHcIcbtJv+5aJ1RmzcKVNnAu1oualuPhCx9fhQPvAgoFk+GScKWSN6FXu/MH/Xx09CNnCKx6DXUtziJiU/KndA6BlQ4rjfmX+LaDjby4sZplH9fR7A1QWuBUKXdeUciwtAEy0/Q5sCCJYEFi4klVad+yhcaVK/FUriJw7BiSmkrm7Flkzp9PxnXX4UpOjnczo89z2AmVba9AzQfOY6OmOoEy+XbIGdOr3akqu0/u7jxYv/X4VgAKMwo7p2y5YsQVJLliN9zU6gvw2pZDvLChms01J3EnulhwyUgWl5VQNi530FUpFiQRLEjMQKHBIK0ffURTZSWeVasJNjTgSk8n86YbnVC5+mpkoKyVEk2Ntc70LNtfgYPOadUUTjsVKtm9P636WOsx3ql1jqt8cOgDvEEvmUmZfKnoS8wuns01hdeQ5c7q5478eTvqmnhxYzWv/ukgHm+ACcPTw1VKETnpg+MztiCJYEFiBiINBGj58EMnVN54k1BTE67sbDJvvoms+fNJnzEDSRwCkwo2HHCGvra9Aoed06opnukcT5m0CDJ7f1p1q7+VDw59QFVNFe/UvkN9ez2JrkRmFc2ivLScawqviVml0uoL8Ponh1i6oZo/VZ/EneBi/qVOlTJjgFcpFiQRLEjMQKc+H83vvUfTypU0v/1HQi0tJOTmkjl3Dlnz55M2fXp8lgiOtRN7nUDZ/oozUzHiXEU/+XbnAsj0/F7vMhgKsvX4VlYfWM3K/Supb68nLyWPW8ffSnlpORfkxG76m08PN/Hihhpe/lMtnvYA4/PDVcq0InIHYJUyIIJEROYB/wdIAH6jqj/v8vwS4JfAwfBDj6vqb8LP/QK4BXABbwJ/paoqIlXAKKAt/Jo5qnr0i9phQWIGk1B7O83r1jmhsqYKbW8nsaCAzHlzyZo/n9TLLhvQ/4vtN0c/dQJl2ytwYrezpsq465xK5eJbIS2317v0h/y8W/suFXsreKfmHQIaYFLeJBaVLmLBuAUxu6q+zRdk5VbnWMpHnzfgTnAx95KRLC4r5qrxeQPm8417kIhIArALuBmoBTYCi1V1R8Q2S4DpqvqdLq+9Gidgrgs/9C7wE1WtCgfJD1S1x8lgQWIGq1BLC56qKpoqK2lZuw71+UgaPZrM+fPImr+AlMmTBsyXTtSowpFtpyqVhgPgSoIJs51jKhcvgJTeB0B9ez0r962kYm8Fn9Z/SpIriVnFs1hUuoirR19Nois2w4q7jnhYuqGalz+qpak9wLj8dO65spg7pxWRnxHfkzAGQpBcBTyiqnPD938CoKqPRWyzhO6D5CrgceBLgABrga+r6k4LEjNUBT0ePG+/7YTK+vcgECBpTAlZ8+c7a6lceGG8mxh9qs5aKttfdX4aayDBDaU3O5XKhfMguffX6nxa/ykVeyp4fd/rNHgbyE/N57bxt1FeWs6EYb07RflctfuDVG47xAsfVrPxQANJCcKcySO5t6yEq8bn4YrD0sEDIUjuAuap6jfD978OzIgMjXCQPAYcw6levqeqNeHnfgV8EydIHlfVvws/XgXkAUHgZeB/6Vk6YUFizjeBhgY8b72Fp7KSlg8+hFAId+kEJ1TmLyB5/Lh4NzH6VKF2k1OlbH8VPIcgMRUunONUKhfMOeuqj135g37WHlxLxZ4K1tWuI6ABLsm7hEWli5g3bl7Mhr52H/GwNHwspbHNz5i8NO65soS7phUxPDN2VcpACJIvA3O7BEmZqv5lxDZ5QLOqekXkW8BXVPUGESnFObZyd3jTN4Efq+paESlU1YMikokTJL9T1d928/4PAQ8BlJSUTPv888+j0k9j4i1w/DhNb7yBZ2UlrR99BKokT5wYrlTm4y4qincToy8Ucq5N2fYK7FgGLcfCqz7OdyqVCTf2aNXHSCfaTvD6vtdZtncZuxt2k+RK4oaSGyifUM5Vo6+KydBXuz/Iqm2HeWFDNRv215PoEuZMHsHishKumZAf9SplIATJWYe2umyfANSraraI/BBIUdWfhZ/7H0C7qv6iy2uW0M3QWFdWkZihwn/kCJ5Vq2hcuZL2Lc6ptClTpjhrqcybR9LIkXFuYQyEgs5V9NtfgR3Loa3eWfXxgpshZyxkjHROKc4YCRkFkDkSklL/7O5U1Rn62usMfZ30nqQgtYBbJ9xK+YRyxg8bH5Nu7TnazIsbqnn5T7U0tPopzk3lnitL+PL0IgoyexeSPTUQgiQRZ7jqRpyzsjYC96rq9ohtRqnqofDt23GqjpkicjfwIDAPZ2hrFfCvQCUwTFWPi0gSsBR4S1X//YvaYkFihqLOGYorK/Hu2AlA6rRpTqjMnUtifu9PpR10gn5n1cdtrzqTSTYfBg2duV1ydjhcwj+ZI0//M2MEZI7Al5TG2oPrnKGvg+sIapAp+VMoLy1n3rh5Mbngsd0fZPX2wyzdUM0H+5wq5aaJI1g8o4RrS/u3Sol7kIQbsQAnABKAp1X1H0XkUWCTqi4XkceAhUAAqAceVtVPw9XJr3HO2lJglap+X0TScQ68J4X3+RbwfVUNflE7LEjMUNcxQ7GnshLv7j3gcpE5Zw55DywhderUeDcvdkJBZ/6v5iPOj+ewEy6eI10eO+KstdJVYopTxWSM5Hh6Dq8nBlnmP8oe/0ncksiNBdMpL13IzLFzSejBWvd9tfdYM/+1sYaXPqqlvsVHUU4q91xZzJenFzMiq+9VyoAIkoHCgsSYU9p37aKxooKTv/8DIY+H1CuuIHfJ/WTeeCOSEJ/1PgYcVWdq/OajEUFzOBwypz+m7Y3scCdRkZHByow0GhMSKAgEWOiDha4cxqWPPjWU1qXCIWNkr4/ddMcbCPLG9iMs3VDNe3tPkOASbry4gMUzSrjuguEknGOVYkESwYLEmDMFm1tofOUV6n/7W/y1tSQVF5N7330Mu+N2XOnp8W7e4OFvC4fLEXyNtVQd/oCKE5t5t/0QIWBqMIHy1nbm1R8lM9TN4ElK9unHazqH1Lo8lpLdozVd9h9v4cWN1by0yTnj6/2f3HjOZ3pZkESwIDHmz9NgEM9bb1P/zDO0bd6MKyuLnLvvJudrXx0aywdHybHWY7y27zUq9lSwt3EvyQnJ3Dj6GsoLrmRGUj4JLUfDQ2lHulQ9RyDoPXOHiSlnVjOdJw1EPJaeD64EfIEQWw+eZNqY3s8A0MGCJIIFiTE90/rxx9Q/+xyeN98El4vsWxaQu2QJKRMnxrtpg5aqsv3EdpbtWcbK/Svx+DyMTB/ZecHjmKwxXV8QHlaLOF4TeTvysfbGM99QXJBecKqaueuZc7pIEyxITmNBYkzv+GpqqH/+eRpfeplQaytpM2eSu+R+Mq67bmhMHhkl3qCXNTVrqNhTwXt17xHSEJcXXM6i0kXMGTOHDHcvv/D9badXNc1HTz+BoOUoPLgGznGtewuSCBYkxpybYFMTJ//wB+qf/x2Bw4dxjx9P7v33k12+EFdKdK5dGCqOth7ltX2vsWzPMvY37iclIYWbxtxEeWk5ZSPLcEn8A9uCJIIFiTF9o34/TatWU//MM7Tv2EFCTg45ixeT89V7SczLi3fzBjVVZevxrVTsqaByfyUev4dR6aNYOGEh5RPKKc4qjlvbLEgiWJAY0z9UldaNG6l/5lma16xB3G6yFt5G3pIlJJeWxrt5g157oJ2qmiqW7V3G+3XvE9IQVxRc4Qx9jZ1DelJsz6azIIlgQWJM//Pu20/9b5+jcVkF2t5O+nXXkrdkCWlXXXX+T20fA0dajrBi3woq9lRwoOkAqYmp3DzmZsonlDN95PSYDH1ZkESwIDEmegINDZx88UXq//MFgsePk3zRReQuWUL2LQuGxvrzUaaqfHL8E5btWcaq/ato9jdTmFHIwgkLuW3CbRRnRm/oy4IkggWJMdEX8vloWvEa9c8+i3f3bhKHDyfnq18l5567SRg2LN7NOy+0B9r5Y/Ufqdhbwft176Mo00dMp7y0nDlj5pCW1Ltp88/GgiSCBYkxsaOqtKx/j/pnnqFl/XokNZVhty8i9777cI8dG+/mnTcOtxxmxd4VVOyt4POmz0lNTGXOmDmUl5YzbcS0fhn6siCJYEFiTHy079pF/bPP0bRiBRoIkHHDDc5EkdOm2XGUfqKqbDm2xRn6OrCKFn8LhRmFlE8oZ2HpQgozCs953xYkESxIjImvwLFj1L/wAieXvkjw5ElSLrmE3AeWkDV3LpIYm7XRh4K2QBtvV79NxZ4KPjz0ISLC219+m/zUc1sywIIkggWJMQNDqK2NxooK6p99Dt+BAySOHkXu177OsC/fRUJmZrybd1451HyIDYc3UF5afs77sCCJYEFizMCioRDNVe9Q/+yztG7YgCs9nWF33UXufV8nqfDch2JM/7IgiWBBYszA1bZtO/XPPUdTZSWokjnnZvIeeIDUKVPi3bQhz4IkggWJMQOf/9AhGv7zP2n4r9+fWnDrgSVk3nCDLbgVJxYkESxIjBk8zlhwq6TEWXDr9kW24FaMWZBEsCAxZvA5Y8Gt7GxyvvIVcr72NZJGFMS7eUOCBUkECxJjBrfTFtxKSCB7wXxbcCsGehokdgK3MWbAS7v8ctIuv7xzwa2TL71MY8Vy0mbOJO+BJaRfe60tuBVHVpEYYwadMxbcmjCB3PvvI3uhLbjVn2xoK4IFiTHnpzMW3MrNdRbcunexLbjVD3oaJFGtBUVknoh8JiJ7RORvunl+iYgcE5HN4Z9vRjz3CxHZLiI7ReT/SpeJeURkuYhsi2b7jTEDmyQlkX3brYx9+SVKfvscqVOncvyJJ9gz+wYO/f3f492zJ95NHBKidoxERBKAJ4CbgVpgo4gsV9UdXTb9L1X9TpfXXg1cA3RckfQucD1QFX7+DqA5Wm03xgwuIkJ6WRnpZWWnFtx6dRkn//CSs+DWAw+QNnOmTRQZJdE82F4G7FHVfQAi8iJQDnQNku4okAK4AQGSgCPh/WQA3wceAn7f/802xgxmyePHMeqRRxj+V3/VueBW9QPfwD1mDIkFBUhyMpKcjCv8pyS7ndvuZCQl+dTtjudSUsL33RGvST79ttuNpKQM2QsnoxkkhUBNxP1aYEY3290pItcBu4DvqWqNqr4vImuAQzhB8riq7gxv/zPgfwOtX/TmIvIQTthQUlLSp44YYwafxJwc8h9+mNxvfIOm117H88YbhFpbCXqa0OM+1Osl5G1Hvc5t9XpRv7+Pb5rohEo3YSPJblzuZCdwOm53budGklMibjth5kpJ7rzdGWQpKYj7zFAjMTFuFVc0g6S7HnU9sr8CWKqqXhH5FvAccIOIlAITgaLwdm+Gw6YJKFXV74nI2C96c1V9EngSnIPt59wLY8yg5kpOZtiddzDszjvOuq2GQp2hEvL6UJ8XbW8/ddvrJeT1ou1e1Be+7e0mlHzdbxdqaSHU0HDqPXzhbbxe1OfrY0ddp6qjcOC4kt2M/cMfon4mWzSDpBaIXEy4CKiL3EBVT0TcfQr45/Dt24EPVLUZQEQqgZmAB5gmIgdw2l4gIlWqOisaHTDGDC3iciGpqZCaSqwHqTQUQv1+J2Da21GfLyLUvF1u+1Bv+6nbkcHVEVDhUIvFei/RfIeNwAUiMg44CNwD3Bu5gYiMUtVD4bsLgY7hq2rgQRF5DKeyuR74V1VdAfxb+LVjgdcsRIwx5wMJVxQkJ5OQlRXv5vRK1IJEVQMi8h1gNZAAPK2q20XkUWCTqi4HvisiC4EAUA8sCb/8JeAGYCvOcNiqcIgYY4wZYOyCRGOMMd0aEBckGmOMOf9ZkBhjjOkTCxJjjDF9YkFijDGmTyxIjDHG9IkFiTHGmD4ZEqf/isgx4PNzfHk+cLwfmzMYWJ+HhqHW56HWX+h7n8eo6vCzbTQkgqQvRGRTT86jPp9Yn4eGodbnodZfiF2fbWjLGGNMn1iQGGOM6RMLkrN7Mt4NiAPr89Aw1Po81PoLMeqzHSMxxhjTJ1aRGGOM6RMLkjARmScin4nIHhH5m26e/5aIbBWRzSLyrohMikc7+9PZ+hyx3V0ioiIyqM946cFnvEREjoU/480i8s14tLM/9eQzFpGviMgOEdkuIi/Euo39rQef879EfMa7RORkPNrZn3rQ5xIRWSMiH4vIJyKyoF8boKpD/gdnvZS9wHjADWwBJnXZJivi9kKcNVLi3vZo9jm8XSawFvgAmB7vdkf5M14CPB7vtsa4zxcAHwM54fsF8W53tPvcZfu/xFkrKe5tj/Ln/CTwcPj2JOBAf7bBKhJHGbBHVfepqg94ESiP3EBVmyLupnPm+vODzVn7HPYz4BdAeywbFwU97e/5pCd9fhB4QlUbAFT1aIzb2N96+zkvBpbGpGXR05M+K9Cx7GI2XZY97ysLEkchUBNxvzb82GlE5Nsishfni/W7MWpbtJy1zyJyOVCsqq/FsmFR0qPPGLgzXPq/JCLFsWla1PSkzxcCF4rIehH5QETmxax10dHTzxkRGQOMA/4Yg3ZFU0/6/AjwNRGpBVbiVGL9xoLEId08dkbFoapPqOoE4MfAT6Pequj6wj6LiAv4F+CvY9ai6OrJZ7wCGKuqU4C3gOei3qro6kmfE3GGt2bh/O/8NyIyLMrtiqYe/S6H3QO8pKrBKLYnFnrS58XAs6paBCwAng//jvcLCxJHLRD5v88ivrj0exFYFNUWRd/Z+pwJXAJUicgBYCawfBAfcD/rZ6yqJ1TVG777FDAtRm2Llp78u64FKlTVr6r7gc9wgmWw6s3v8j0M/mEt6Fmf/xvwewBVfR9IwZmHq19YkDg2AheIyDgRceP8A1seuYGIRP5y3QLsjmH7ouEL+6yqjaqar6pjVXUszsH2haq6KT7N7bOefMajIu4uBHbGsH3RcNY+A8uA2QAiko8z1LUvpq3sXz3pMyJyEZADvB/j9kVDT/pcDdwIICITcYLkWH81ILG/djSYqWpARL4DrMY5A+JpVd0uIo8Cm1R1OfAdEbkJ8AMNwP3xa3Hf9bDP540e9ve7IrIQCAD1OGdxDVo97PNqYI6I7ACCwA9V9UT8Wt03vfh3vRh4UcOnMQ1mPezzXwNPicj3cIa9lvRn3+3KdmOMMX1iQ1vGGGP6xILEGGNMn1iQGGOM6RMLEmOMfCrwGgAAArVJREFUMX1iQWKMMaZPLEiM6SMReUREfjAA2nEgfC2IMTFlQWKMMaZPLEiM6YaIpIvI6yKyRUS2icjdkf/jF5HpIlIV8ZKpIvJHEdktIg+GtxklImvD615sE5Frw4//m4hsCq//8Q8R73lARP5JRN4PP3+FiKwWkb0i8q3wNrPC+3w1vIbIv3c3Z5KIfE1ENoTf+/+JSEI0/77M0GZBYkz35gF1qjpVVS8BVp1l+yk4U+dcBfwPERkN3AusVtXLgKnA5vC2f6eq08OvuV5EpkTsp0ZVrwLWAc8Cd+HMc/ZoxDZlOFcqXwpMAO6IbEh4Coy7gWvC7x0EvtqLvhvTKzZFijHd2wr8SkT+GXhNVdeJdDfJaqcKVW0D2kRkDc6X/UbgaRFJApapakeQfEVEHsL5/RuFs9DQJ+HnOqbw2ApkqKoH8IhIe8SsvBtUdR+AiCwFvgS8FNGWG3EmnNwYbnMqMNjXGTEDmAWJMd1Q1V0iMg1nyu3HROQNnDm4Oqr4lK4vOXMXulZErsOpVJ4XkV/iVBo/AK5U1QYRebbLvjpmHw5F3O643/H7esZ7dbkvwHOq+pOzdNOYfmFDW8Z0Izw01aqqvwN+BVwBHODU1PJ3dnlJuYikiEgeztoeG8MLJx1V1aeA/wjvIwtoARpFZAQw/xyaVxae6dWFM4T1bpfn3wbuEpGCcF9yw20xJiqsIjGme5cCvxSREM6Mzw/jDBH9h4j8LfBhl+03AK8DJcDPVLVORO4HfigifqAZuE9V94vIx8B2nOna159D294Hfh5u41rg1cgnVXWHiPwUeCMcNn7g28Dn5/BexpyVzf5rzCAiIrOAH6jqrfFuizEdbGjLGGNMn1hFYowxpk+sIjHGGNMnFiTGGGP6xILEGGNMn1iQGGOM6RMLEmOMMX1iQWKMMaZP/j82vd7vkEBy6gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c18ced7eb8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch4.best_score_, gsearch4.best_params_))\n",
    "test_means = gsearch4.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch4.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch4.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch4.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch4.cv_results_).to_csv('subsampleh_colsample_bytree_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(colsample_bytree), len(subsample))\n",
    "train_scores = np.array(train_means).reshape(len(colsample_bytree), len(subsample))\n",
    "\n",
    "for i, value in enumerate(colsample_bytree):\n",
    "    pyplot.plot(subsample, -test_scores[i], label= 'test_colsample_bytree:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'subsample' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'subsample_vs_colsample_bytree1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.5、正则参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [1.5, 2], 'reg_lambda': [0.5, 1, 2]}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#reg_alpha = [1e-3, 1e-2, 0.05, 0.1]    #default = 0\n",
    "#reg_lambda = [1e-3, 1e-2, 0.05, 0.1]   #default = 1\n",
    "\n",
    "reg_alpha = [ 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58355, std: 0.00378, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58338, std: 0.00374, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58345, std: 0.00367, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58341, std: 0.00361, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58359, std: 0.00343, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58369, std: 0.00371, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       " -0.5833801280132562)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.9,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch5 = GridSearchCV(xgb5, param_grid = param_test5, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5.fit(X_train , y_train)\n",
    "\n",
    "gsearch5.grid_scores_, gsearch5.best_params_,     gsearch5.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3、保存模型测试结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.9, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=5, min_child_weight=1, missing=None, n_estimators=220,\n",
       "       n_jobs=1, nthread=None, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=1.5, reg_lambda=1, scale_pos_weight=1, seed=3,\n",
       "       silent=True, subsample=0.8)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_last = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=220,  \n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.9,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3,\n",
    "        reg_alpha=1.5, \n",
    "        reg_lambda= 1)\n",
    "xgb_last.fit(X_train, y_train, eval_metric='mlogloss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_predprob = xgb_last.predict_proba(data_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "train_pred = xgb_last.predict(data_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(train_pred)\n",
    "df.to_csv('test_file.csv',header=['pred'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
